update structure

proxypool/db.py

@@ -33,7 +33,8 @@ class RedisClient(object):
         :param score:
         :return:
         """
-        return self.db.zadd(REDIS_KEY, score, proxy)
+        if self.db.zscore(proxy):
+            return self.db.zadd(REDIS_KEY, score, proxy)
     
     def random(self):
         """

proxypool/getter.py

@@ -7,7 +7,6 @@ from proxypool.setting import *
 class Getter():
     def __init__(self):
         self.redis = RedisClient()
-        self.tester = Tester()
         self.crawler = Crawler()
     
     def is_over_threshold(self):
@@ -21,19 +20,10 @@ class Getter():
     
     def run(self):
         print('获取器开始执行')
-        proxy_count = 0
         if not self.is_over_threshold():
             for callback_label in range(self.crawler.__CrawlFuncCount__):
                 callback = self.crawler.__CrawlFunc__[callback_label]
                 # 获取代理
                 proxies = self.crawler.get_proxies(callback)
-                # 设置代理并测试
-                self.tester.set_proxies(proxies)
-                self.tester.run()
-                proxy_count += len(proxies)
-                if self.is_over_threshold():
-                    print('代理池已满，暂停抓取')
-                    break
-            if proxy_count == 0:
-                # 代理池枯竭
-                print('代理池已枯竭')
+                for proxy in proxies:
+                    self.redis.add(proxy)

proxypool/scheduler.py

@@ -11,18 +11,9 @@ class Scheduler():
         """
         定时测试代理
         """
-        redis = RedisClient()
         tester = Tester()
         while True:
             print('测试器开始运行')
-            count = redis.count()
-            if count == 0:
-                print('代理池已枯竭，等待添加代理')
-                time.sleep(cycle)
-                continue
-            proxies = redis.all()
-            tester.set_proxies(proxies)
-            print('开始检测全部代理')
             tester.run()
             time.sleep(cycle)
     

proxypool/tester.py

@@ -11,17 +11,8 @@ from proxypool.setting import *
 
 class Tester(object):
     def __init__(self):
-        self.proxies = None
         self.redis = RedisClient()
     
-    def set_proxies(self, proxies):
-        """
-        设置代理
-        :param proxies:
-        :return:
-        """
-        self.proxies = proxies
-    
     async def test_single_proxy(self, proxy):
         """
         测试单个代理
@@ -54,8 +45,9 @@ class Tester(object):
         """
         print('测试器开始运行')
         try:
+            proxies = self.redis.all()
             loop = asyncio.get_event_loop()
-            tasks = [self.test_single_proxy(proxy) for proxy in self.proxies]
+            tasks = [self.test_single_proxy(proxy) for proxy in proxies]
             loop.run_until_complete(asyncio.wait(tasks))
         except Exception as e:
             print('测试器发生错误', e.args)