merge

proxypool/crawler.py

@@ -1,4 +1,4 @@
-import json
+ccimport json
 import re
 from .utils import get_page
 from pyquery import PyQuery as pq
@@ -31,7 +31,7 @@ class Crawler(object, metaclass=ProxyMetaclass):
             urls = html.split('\n')
             for url in urls:
                 yield url
-
+          
     def crawl_daili66(self, page_count=4):
         """
         获取代理66

proxypool/importer.py

@@ -1,5 +1,3 @@
-import requests
-
 from proxypool.db import RedisClient
 
 conn = RedisClient()
@@ -14,10 +12,10 @@ def set(proxy):
 def scan():
     print('请输入代理, 输入exit退出读入')
     while True:
-        account = input()
-        if account == 'exit':
+        proxy = input()
+        if proxy == 'exit':
             break
-        set(account)
+        set(proxy)
 
 
 if __name__ == '__main__':

proxypool/setting.py

@@ -25,7 +25,7 @@ TESTER_CYCLE = 20
 GETTER_CYCLE = 300
 
 # 测试API，建议抓哪个网站测哪个
-TEST_URL = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=2145291155&containerid=1076032145291155&page=14'
+TEST_URL = 'http://www.baidu.com'
 
 # API配置
 API_HOST = '0.0.0.0'
@@ -35,3 +35,6 @@ API_PORT = 5555
 TESTER_ENABLED = True
 GETTER_ENABLED = True
 API_ENABLED = True
+
+# 最大批测试量
+BATCH_TEST_SIZE = 100

proxypool/tester.py

@@ -1,6 +1,10 @@
 import asyncio
 import aiohttp
 import time
+<<<<<<< HEAD
+=======
+
+>>>>>>> dbb6bb89903b1cd158470ac472a5a930f7f0978b
 try:
     from aiohttp import ClientError
 except:
@@ -12,7 +16,7 @@ from proxypool.setting import *
 class Tester(object):
     def __init__(self):
         self.redis = RedisClient()
-
+    
     async def test_single_proxy(self, proxy):
         """
         测试单个代理
@@ -26,17 +30,17 @@ class Tester(object):
                     proxy = proxy.decode('utf-8')
                 real_proxy = 'http://' + proxy
                 print('正在测试', proxy)
-                async with session.get(TEST_URL, proxy=real_proxy, timeout=15) as response:
+                async with session.get(TEST_URL, proxy=real_proxy, timeout=15, allow_redirects=False) as response:
                     if response.status in VALID_STATUS_CODES:
                         self.redis.max(proxy)
                         print('代理可用', proxy)
                     else:
                         self.redis.decrease(proxy)
-                        print('请求响应码不合法，IP', proxy)
+                        print('请求响应码不合法 ', response.status, 'IP', proxy)
             except (ClientError, aiohttp.client_exceptions.ClientConnectorError, asyncio.TimeoutError, AttributeError):
                 self.redis.decrease(proxy)
                 print('代理请求失败', proxy)
-
+    
     def run(self):
         """
         测试主函数
@@ -46,9 +50,8 @@ class Tester(object):
         try:
             proxies = self.redis.all()
             loop = asyncio.get_event_loop()
-            batch_size = 100
-            for i in range(0, len(proxies), batch_size):
-                test_proxies = proxies[i:i + batch_size]
+            for i in range(0, len(proxies), BATCH_TEST_SIZE):
+                test_proxies = proxies[i:i + BATCH_TEST_SIZE]
                 tasks = [self.test_single_proxy(proxy) for proxy in test_proxies]
                 loop.run_until_complete(asyncio.wait(tasks))
                 time.sleep(5)