Selaa lähdekoodia

update tester

Germey 6 vuotta sitten
vanhempi
sitoutus
fab05831f4
2 muutettua tiedostoa jossa 4 lisäystä ja 3 poistoa
  1. 1 1
      proxypool/crawlers/base.py
  2. 3 2
      proxypool/processors/tester.py

+ 1 - 1
proxypool/crawlers/base.py

@@ -24,5 +24,5 @@ class BaseCrawler(object):
             logger.info(f'fetching {url}')
             html = self.fetch(url)
             for proxy in self.parse(html):
-                logger.info(f'fetched proxy {proxy.string()}')
+                logger.info(f'fetched proxy {proxy.string()} from {url}')
                 yield proxy

+ 3 - 2
proxypool/processors/tester.py

@@ -4,7 +4,7 @@ from loguru import logger
 from proxypool.schemas import Proxy
 from proxypool.storages.redis import RedisClient
 from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS
-from aiohttp import ClientProxyConnectionError, ServerDisconnectedError
+from aiohttp import ClientProxyConnectionError, ServerDisconnectedError, ClientOSError
 from asyncio import TimeoutError
 
 
@@ -12,7 +12,8 @@ EXCEPTIONS = (
     ClientProxyConnectionError,
     ConnectionRefusedError,
     TimeoutError,
-    ServerDisconnectedError
+    ServerDisconnectedError,
+    ClientOSError
 )