Merge pull request #153 from highroom/branch1

增加了对需要FQ代理网站的配置

Config.ini

@@ -30,3 +30,7 @@ freeProxyWallThird = 1
 ; API接口配置 http://127.0.0.1:5010
 ip = 0.0.0.0
 port = 5010
+
+[WallProxy]
+; fq代理配置
+; proxy = 127.0.0.1:1080

ProxyGetter/getFreeProxy.py

@@ -14,6 +14,12 @@
 import re
 import sys
 import requests
+import os
+
+try:
+    from configparser import ConfigParser  # py3
+except:
+    from ConfigParser import ConfigParser  # py2
 
 
 try:
@@ -48,6 +54,15 @@ class GetFreeProxy(object):
     """
     proxy getter
     """
+    pwd = os.path.split(os.path.realpath(__file__))[0]
+    config_path = os.path.join(os.path.split(pwd)[0], 'Config.ini')
+    config_file = ConfigParser()
+    config_file.read(config_path)
+    if config_file.has_option('WallProxy', 'proxy'):
+        WallProxy = config_file.get('WallProxy', 'proxy')
+        wall_proxies = {"http": "http://{}".format(WallProxy), "https": "https://{}".format(WallProxy)}
+    else:
+        wall_proxies = None   
 
     def __init__(self):
         pass
@@ -277,10 +292,17 @@ class GetFreeProxy(object):
         墙外网站 cn-proxy
         :return:
         """
+        kwargs = {}
+        if GetFreeProxy.wall_proxies:
+            kwargs['proxies'] = GetFreeProxy.wall_proxies
+        else:
+            return
+
         urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']
         request = WebRequest()
         for url in urls:
-            r = request.get(url)
+            kwargs['url'] = url
+            r = request.get(**kwargs)
             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
@@ -291,21 +313,35 @@ class GetFreeProxy(object):
         https://proxy-list.org/english/index.php
         :return:
         """
+        kwargs = {}
+        if GetFreeProxy.wall_proxies:
+            kwargs['proxies'] = GetFreeProxy.wall_proxies
+        else:
+            return
         urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)]
         request = WebRequest()
         import base64
         for url in urls:
-            r = request.get(url)
+            kwargs['url'] = url
+            r = request.get(**kwargs)
             proxies = re.findall(r"Proxy\('(.*?)'\)", r.text)
             for proxy in proxies:
                 yield base64.b64decode(proxy).decode()
 
     @staticmethod
     def freeProxyWallThird():
+    
+        kwargs = {}
+        if GetFreeProxy.wall_proxies:
+            kwargs['proxies'] = GetFreeProxy.wall_proxies
+        else:
+            return
+    
         urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1']
         request = WebRequest()
         for url in urls:
-            r = request.get(url)
+            kwargs['url'] = url
+            r = request.get(**kwargs)
             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
@@ -356,3 +392,5 @@ if __name__ == '__main__':
     # test_batch(gg.freeProxyWallSecond())
 
     # test_batch(gg.freeProxyWallThird())
+    for e in gg.freeProxyWallThird():
+        print(e)

Util/WebRequest.py

@@ -70,7 +70,7 @@ class WebRequest(object):
             headers.update(header)
         while True:
             try:
-                html = requests.get(url, headers=headers, timeout=timeout)
+                html = requests.get(url, headers=headers, timeout=timeout, **kwargs)
                 if any(f in html.content for f in retry_flag):
                     raise Exception
                 return html