修复is_valid_proxy函数的bug，add: 站大爷crawler (#68)

* fix: 修复is_valid_proxy函数的bug，add: 站大爷crawler

* fix: 修改__init__中 站大爷 detailcrawler的判断逻辑，使其更为通用

proxypool/crawlers/__init__.py

@@ -1,14 +1,15 @@
 import pkgutil
 from .base import BaseCrawler
+from .public.zhandaye import ZhandayeDetailCrawler
 import inspect
 
-
 # load classes subclass of BaseCrawler
 classes = []
 for loader, name, is_pkg in pkgutil.walk_packages(__path__):
     module = loader.find_module(name).load_module(name)
     for name, value in inspect.getmembers(module):
         globals()[name] = value
-        if inspect.isclass(value) and issubclass(value, BaseCrawler) and value is not BaseCrawler:
+        if inspect.isclass(value) and issubclass(value, BaseCrawler) and value is not BaseCrawler \
+                and not getattr(value, 'ignore', False):
             classes.append(value)
 __all__ = __ALL__ = classes

proxypool/crawlers/public/zhandaye.py

@@ -0,0 +1,59 @@
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from loguru import logger
+
+
+BASE_URL = 'https://www.zdaye.com/dayProxy/{page}.html'
+MAX_PAGE = 5
+
+class ZhandayeCrawler(BaseCrawler):
+    """
+    zhandaye crawler, https://www.zdaye.com/dayProxy/
+    """
+    urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE)]
+
+    def crawl(self):
+        for url in self.urls:
+            logger.info(f'fetching {url}')
+            if not self.headers:
+                html = self.fetch(url)
+            else:
+                html = self.fetch(url, headers=self.headers)
+            self.parse(html)
+
+    def parse(self, html):
+        """
+        parse html file to get proxies
+        :return:
+        """
+        doc = pq(html)
+        for item in doc('#J_posts_list .thread_item div div p a').items():
+            post = 'https://www.zdaye.com' + item.attr('href')
+            logger.info(f'get detail url: {post}')
+            ZhandayeDetailCrawler(post).crawl()
+
+
+class ZhandayeDetailCrawler(BaseCrawler):
+    urls = []
+    ignore = True
+
+    def __init__(self, url):
+        self.urls.append(url)
+        super().__init__()
+
+    def parse(self, html):
+        doc = pq(html)
+        trs = doc('.cont br').items()
+        for tr in trs:
+            line = tr[0].tail
+            host = line.split(':')[0]
+            port = line.split(':')[1][:4]
+            yield Proxy(host=host, port=port)
+
+
+
+if __name__ == '__main__':
+    crawler = ZhandayeCrawler()
+    for proxy in crawler.crawl():
+        print(proxy)

proxypool/utils/proxy.py

@@ -1,14 +1,30 @@
 from proxypool.schemas import Proxy
-import re
 
 
 def is_valid_proxy(data):
-    """
-    is data is valid proxy format
-    :param data:
-    :return:
-    """
-    return re.match('\d+\.\d+\.\d+\.\d+\:\d+', data)
+    if data.__contains__(':'):
+        ip = data.split(':')[0]
+        port = data.split(':')[1]
+        return is_ip_valid(ip) and is_port_valid(port)
+    else:
+        return is_ip_valid(data)
+
+
+def is_ip_valid(ip):
+    a = ip.split('.')
+    if len(a) != 4:
+        return False
+    for x in a:
+        if not x.isdigit():
+            return False
+        i = int(x)
+        if i < 0 or i > 255:
+            return False
+    return True
+
+
+def is_port_valid(port):
+    return port.isdigit()
 
 
 def convert_proxy_or_proxies(data):