update structure

.gitignore

@@ -2,4 +2,5 @@
 *.pyc
 *.db
 venv
-/.idea
+/.idea
+*.log

LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright {yyyy} {name of copyright owner}
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

README.md

@@ -1,52 +1,76 @@
 # ProxyPool
 
-## 安装
+## Requirements
 
-### 安装Python
+* Docker 
 
-至少Python3.5以上
+  or 
 
-### 安装Redis
+* Python: >=3.6
+* Redis
+* Environment: Virtual Env
 
-安装好之后将Redis服务开启
+## Run with Docker
 
-### 配置代理池
-
-```
-cd proxypool
+```shell script
+docker-compose up
 ```
 
-进入proxypool目录，修改settings.py文件
+## Run without Docker
 
-PASSWORD为Redis密码，如果为空，则设置为None
+Here are steps to run ProxyPool.
 
-#### 安装依赖
+### Install Redis
 
-```
-pip3 install -r requirements.txt
+You need to install Redis locally or get a Redis server firstly.
+
+Next set Redis environment:
+
+```shell script
+export REDIS_HOST='localhost'
+export REDIS_PORT=6379
+export REDIS_PASSWORD='foobar'
 ```
 
-#### 打开代理池和API
+Also you can just set the Redis Connection String:
 
+```shell script
+export REDIS_CONNECTION_STRING='redis://[password]@host:port'
 ```
-python3 run.py
+
+You can choose one method of above to set Redis environment.
+
+### Clone ProxyPool
+
+```shell script
+https://github.com/Python3WebSpider/ProxyPool
+cd ProxyPool
 ```
 
-## 获取代理
+### Install Requirements
+
+```shell script
+pip3 install -r requirements.txt
+```
 
+### Run ProxyPool
 
-利用requests获取方法如下
+You can run all of the processors including Getter、Tester、
+Server:
 
-```python
-import requests
+```shell script
+python3 run.py
+```
 
-PROXY_POOL_URL = 'http://localhost:5555/random'
+or run with args to run specific processor:
 
-def get_proxy():
-    try:
-        response = requests.get(PROXY_POOL_URL)
-        if response.status_code == 200:
-            return response.text
-    except ConnectionError:
-        return None
+```shell script
+python3 run.py --processor getter
+python3 run.py --processor tester
+python3 run.py --processor server
 ```
+
+### Usage
+
+After running the ProxyPool, you can visit 
+[http://localhost:5555/random](http://localhost:5555/) to access random proxy.  

importer.py

@@ -1,4 +0,0 @@
-from proxypool.importer import scan
-
-if __name__ == '__main__':
-    scan()

proxy provider.txt

@@ -1,7 +0,0 @@
-代理：
-https://proxy.mimvp.com/free.php?proxy=in_hp
-http://www.coobobo.com/free-http-proxy
-http://ip.zdaye.com/
-http://www.mayidaili.com/free/anonymous/%E9%AB%98%E5%8C%BF
-http://http.taiyangruanjian.com/
-http://http.zhimaruanjian.com/

proxypool/.gitignore

@@ -0,0 +1,134 @@
+# Created by .ignore support plugin (hsz.mobi)
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.idea/
+*.log

proxypool/crawlers/__init__.py

@@ -6,5 +6,5 @@ from .iphai import IPHaiCrawler
 __all__ = [
     Daili66Crawler,
     IP3366Crawler,
-    IP3366Crawler
+    IPHaiCrawler
 ]

proxypool/crawlers/base.py

@@ -14,7 +14,7 @@ class BaseCrawler(object):
                 return response.text
         except requests.ConnectionError:
             return
-        
+    
     @logger.catch
     def crawl(self):
         """
@@ -24,4 +24,5 @@ class BaseCrawler(object):
             logger.info(f'fetching {url}')
             html = self.fetch(url)
             for proxy in self.parse(html):
+                logger.info(f'fetched proxy {proxy.string()}')
                 yield proxy

proxypool/crawlers/daili66.py

@@ -1,9 +1,9 @@
 from pyquery import PyQuery as pq
-from proxypool.proxy import Proxy
+from proxypool.schemas.proxy import Proxy
 from proxypool.crawlers.base import BaseCrawler
 
 
-BASE_URL = 'http://www.636ip.cn/{page}.html'
+BASE_URL = 'http://www.664ip.cn/{page}.html'
 MAX_PAGE = 5
 
 class Daili66Crawler(BaseCrawler):

proxypool/crawlers/ip3366.py

@@ -1,5 +1,5 @@
 from proxypool.crawlers.base import BaseCrawler
-from proxypool.proxy import Proxy
+from proxypool.schemas.proxy import Proxy
 import re
 
 

proxypool/crawlers/iphai.py

@@ -1,5 +1,5 @@
 from proxypool.crawlers.base import BaseCrawler
-from proxypool.proxy import Proxy
+from proxypool.schemas.proxy import Proxy
 import re
 
 

proxypool/exceptions/__init__.py

@@ -0,0 +1 @@
+from .empty import PoolEmptyException

proxypool/error.py → proxypool/exceptions/empty.py

@@ -1,4 +1,4 @@
-class PoolEmptyError(Exception):
+class PoolEmptyException(Exception):
     def __str__(self):
         """
         proxypool is used out

proxypool/getter.py → proxypool/processors/getter.py

@@ -1,14 +1,14 @@
-from proxypool.db import RedisClient
+from loguru import logger
+from proxypool.storages.redis import RedisClient
 from proxypool.setting import PROXY_NUMBER_MAX
 from proxypool.crawlers import __all__ as crawlers_cls
 
 
-class Getter():
+class Getter(object):
     """
     getter of proxypool
     """
     
-    
     def __init__(self):
         """
         init db and crawlers
@@ -17,7 +17,6 @@ class Getter():
         self.crawlers_cls = crawlers_cls
         self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls]
     
-    
     def is_full(self):
         """
         if proxypool if full
@@ -25,7 +24,7 @@ class Getter():
         """
         return self.redis.count() >= PROXY_NUMBER_MAX
     
-    
+    @logger.catch
     def run(self):
         """
         run crawlers to get proxy
@@ -35,7 +34,6 @@ class Getter():
             return
         for crawler in self.crawlers:
             for proxy in crawler.crawl():
-                print('proxy', proxy)
                 self.redis.add(proxy)
 
 

proxypool/api.py → proxypool/processors/server.py

@@ -1,11 +1,13 @@
 from flask import Flask, g
-from .db import RedisClient
-from .setting import API_HOST, API_PORT
+from proxypool.storages.redis import RedisClient
+from proxypool.setting import API_HOST, API_PORT, API_THREADED
+
 
 __all__ = ['app']
 
 app = Flask(__name__)
 
+
 def get_conn():
     """
     get redis client object
@@ -15,27 +17,35 @@ def get_conn():
         g.redis = RedisClient()
     return g.redis
 
+
 @app.route('/')
 def index():
+    """
+    get home page, you can define your own templates
+    :return:
+    """
     return '<h2>Welcome to Proxy Pool System</h2>'
 
+
 @app.route('/random')
 def get_proxy():
     """
     get a random proxy
-    :return: 随机代理
+    :return: get a random proxy
     """
     conn = get_conn()
-    return conn.random()
+    return conn.random().string()
+
 
 @app.route('/count')
-def get_counts():
+def get_count():
     """
     get the count of proxies
-    :return: 代理池总量
+    :return: count, int
     """
     conn = get_conn()
     return str(conn.count())
 
+
 if __name__ == '__main__':
-    app.run(host=API_HOST, port=API_PORT, threaded=True)
+    app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)

proxypool/processors/tester.py

@@ -0,0 +1,74 @@
+import asyncio
+import aiohttp
+from loguru import logger
+from proxypool.schemas import Proxy
+from proxypool.storages.redis import RedisClient
+from proxypool.setting import TEST_TIMEOUT, TEST_BATCH, TEST_URL, TEST_VALID_STATUS
+from aiohttp import ClientProxyConnectionError, ServerDisconnectedError
+from asyncio import TimeoutError
+
+
+EXCEPTIONS = (
+    ClientProxyConnectionError,
+    ConnectionRefusedError,
+    TimeoutError,
+    ServerDisconnectedError
+)
+
+
+class Tester(object):
+    """
+    tester for testing proxies in queue
+    """
+    
+    def __init__(self):
+        """
+        init redis
+        """
+        self.redis = RedisClient()
+        self.loop = asyncio.get_event_loop()
+    
+    async def test(self, proxy: Proxy):
+        """
+        test single proxy
+        :param proxy: Proxy object
+        :return:
+        """
+        async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session:
+            try:
+                logger.debug(f'testing {proxy.string()}')
+                async with session.get(TEST_URL, proxy=f'http://{proxy.string()}', timeout=TEST_TIMEOUT,
+                                       allow_redirects=False) as response:
+                    if response.status in TEST_VALID_STATUS:
+                        self.redis.max(proxy)
+                        logger.debug(f'proxy {proxy.string()} is valid, set max score')
+                    else:
+                        self.redis.decrease(proxy)
+                        logger.debug(f'proxy {proxy.string()} is invalid, decrease score')
+            except EXCEPTIONS:
+                self.redis.decrease(proxy)
+                logger.debug(f'proxy {proxy.string()} is invalid, decrease score')
+    
+    @logger.catch
+    def run(self):
+        """
+        test main method
+        :return:
+        """
+        # event loop of aiohttp
+        logger.info('stating tester...')
+        count = self.redis.count()
+        logger.debug(f'{count} proxies to test')
+        for i in range(0, count, TEST_BATCH):
+            # start end end offset
+            start, end = i, min(i + TEST_BATCH, count)
+            logger.debug(f'testing proxies from {start} to {end} indices')
+            proxies = self.redis.batch(start, end)
+            tasks = [self.test(proxy) for proxy in proxies]
+            # run tasks using event loop
+            self.loop.run_until_complete(asyncio.wait(tasks))
+
+
+if __name__ == '__main__':
+    tester = Tester()
+    tester.run()

proxypool/scheduler.py

@@ -1,50 +1,92 @@
 import time
-from multiprocessing import Process
-from proxypool.api import app
-from proxypool.getter import Getter
-from proxypool.tester import Tester
-from proxypool.db import RedisClient
-from proxypool.setting import *
+import multiprocessing
+from proxypool.processors.server import app
+from proxypool.processors.getter import Getter
+from proxypool.processors.tester import Tester
+from proxypool.setting import CYCLE_GETTER, CYCLE_TESTER, API_HOST, API_THREADED, API_PORT, ENABLE_SERVER, \
+    ENABLE_GETTER, ENABLE_TESTER, IS_WINDOWS
+from loguru import logger
+
+
+if IS_WINDOWS:
+    multiprocessing.freeze_support()
+
+tester_process, getter_process, server_process = None, None, None
 
 
 class Scheduler():
-    def schedule_tester(self, cycle=TESTER_CYCLE):
+    """
+    scheduler
+    """
+    
+    def run_tester(self, cycle=CYCLE_TESTER):
         """
-        定时测试代理
+        run tester
         """
         tester = Tester()
+        loop = 0
         while True:
-            print('测试器开始运行')
+            logger.debug(f'tester loop {loop} start...')
             tester.run()
+            loop += 1
             time.sleep(cycle)
     
-    def schedule_getter(self, cycle=GETTER_CYCLE):
+    def run_getter(self, cycle=CYCLE_GETTER):
         """
-        定时获取代理
+        run getter
         """
         getter = Getter()
+        loop = 0
         while True:
-            print('开始抓取代理')
+            logger.debug(f'getter loop {loop} start...')
             getter.run()
+            loop += 1
             time.sleep(cycle)
     
-    def schedule_api(self):
+    def run_server(self):
         """
-        开启API
+        run server for api
         """
-        app.run(API_HOST, API_PORT)
+        app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
     
     def run(self):
-        print('代理池开始运行')
-        
-        if TESTER_ENABLED:
-            tester_process = Process(target=self.schedule_tester)
-            tester_process.start()
-        
-        if GETTER_ENABLED:
-            getter_process = Process(target=self.schedule_getter)
-            getter_process.start()
-        
-        if API_ENABLED:
-            api_process = Process(target=self.schedule_api)
-            api_process.start()
+        global tester_process, getter_process, server_process
+        try:
+            logger.info('starting proxypool...')
+            if ENABLE_TESTER:
+                tester_process = multiprocessing.Process(target=self.run_tester)
+                logger.info(f'starting tester, pid {tester_process.pid}...')
+                tester_process.start()
+            
+            if ENABLE_GETTER:
+                getter_process = multiprocessing.Process(target=self.run_getter)
+                logger.info(f'starting getter, pid{getter_process.pid}...')
+                getter_process.start()
+            
+            if ENABLE_SERVER:
+                server_process = multiprocessing.Process(target=self.run_server)
+                logger.info(f'starting server, pid{server_process.pid}...')
+                server_process.start()
+            
+            tester_process.join()
+            getter_process.join()
+            server_process.join()
+        except KeyboardInterrupt:
+            logger.info('received keyboard interrupt signal')
+            tester_process.terminate()
+            getter_process.terminate()
+            server_process.terminate()
+        finally:
+            # must call join method before calling is_alive
+            tester_process.join()
+            getter_process.join()
+            server_process.join()
+            logger.info(f'tester is {"alive" if tester_process.is_alive() else "dead"}')
+            logger.info(f'getter is {"alive" if getter_process.is_alive() else "dead"}')
+            logger.info(f'server is {"alive" if server_process.is_alive() else "dead"}')
+            logger.info('proxy terminated')
+
+
+if __name__ == '__main__':
+    scheduler = Scheduler()
+    scheduler.run()

proxypool/schemas/__init__.py

@@ -0,0 +1 @@
+from .proxy import Proxy

proxypool/proxy.py → proxypool/schemas/proxy.py

@@ -10,9 +10,17 @@ class Proxy(object):
     port = attr(type=int, default=None)
     
     def __str__(self):
+        """
+        to string, for print
+        :return:
+        """
         return f'{self.host}:{self.port}'
     
     def string(self):
+        """
+        to string
+        :return: <host>:<port>
+        """
         return self.__str__()
 
 

proxypool/setting.py

@@ -1,10 +1,19 @@
+import platform
+from os.path import dirname, abspath, join
 from environs import Env
-from proxypool.utils import parse_redis_connection_string
+from loguru import logger
 
 
 env = Env()
 env.read_env()
 
+# definition of flags
+IS_WINDOWS = platform.system().lower() == 'windows'
+
+# definition of dirs
+ROOT_DIR = dirname(dirname(abspath(__file__)))
+LOG_DIR = join(ROOT_DIR, env.str('LOG_DIR', 'logs'))
+
 # definition of environments
 DEV_MODE, TEST_MODE, PROD_MODE = 'dev', 'test', 'prod'
 APP_ENV = env.str('APP_ENV', DEV_MODE).lower()
@@ -26,7 +35,7 @@ REDIS_PASSWORD = env.str('REDIS_PASSWORD', None)
 #     REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING)
 
 # redis hash table key name
-REDIS_KEY = env.str('REDIS_HOST', 'proxies')
+REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal')
 
 # definition of proxy scores
 PROXY_SCORE_MAX = 100
@@ -59,4 +68,7 @@ API_THREADED = env.bool('API_THREADED', True)
 # flags of enable
 ENABLE_TESTER = env.bool('ENABLE_TESTER', True)
 ENABLE_GETTER = env.bool('ENABLE_GETTER', True)
-ENABLE_API = env.bool('ENABLE_API', True)
+ENABLE_SERVER = env.bool('ENABLE_SERVER', True)
+
+logger.add(env.str('LOG_RUNTIME_FILE', 'runtime.log'), level='DEBUG', rotation='1 week', retention='20 days')
+logger.add(env.str('LOG_ERROR_FILE', 'error.log'), level='ERROR', rotation='1 week')

proxypool/db.py → proxypool/storages/redis.py

@@ -1,13 +1,12 @@
 import redis
-from proxypool.error import PoolEmptyError
-from proxypool.proxy import Proxy
-from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_KEY
-from proxypool.setting import PROXY_SCORE_MAX, PROXY_SCORE_MIN, PROXY_SCORE_INIT
+from proxypool.exceptions import PoolEmptyException
+from proxypool.schemas.proxy import Proxy
+from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \
+    PROXY_SCORE_INIT
 from random import choice
 from typing import List
 from loguru import logger
-
-from proxypool.utils import is_valid_proxy, convert_proxy_or_proxies
+from proxypool.utils.proxy import is_valid_proxy, convert_proxy_or_proxies
 
 
 REDIS_CLIENT_VERSION = redis.__version__
@@ -38,7 +37,7 @@ class RedisClient(object):
         if not is_valid_proxy(f'{proxy.host}:{proxy.port}'):
             logger.info(f'invalid proxy {proxy}, throw it')
             return
-        if not self.db.zscore(REDIS_KEY, proxy):
+        if not self.exists(proxy):
             if IS_REDIS_VERSION_2:
                 return self.db.zadd(REDIS_KEY, score, proxy)
             return self.db.zadd(REDIS_KEY, {proxy: score})
@@ -60,7 +59,7 @@ class RedisClient(object):
         if len(proxies):
             return convert_proxy_or_proxies(choice(proxies))
         # else raise error
-        raise PoolEmptyError
+        raise PoolEmptyException
     
     def decrease(self, proxy: Proxy) -> int:
         """
@@ -86,7 +85,7 @@ class RedisClient(object):
         :param proxy: proxy
         :return: if exists, bool
         """
-        return not self.db.zscore(REDIS_KEY, proxy.string()) == None
+        return not self.db.zscore(REDIS_KEY, proxy.string()) is None
     
     def max(self, proxy: Proxy) -> int:
         """
@@ -125,7 +124,5 @@ class RedisClient(object):
 
 if __name__ == '__main__':
     conn = RedisClient()
-    result = conn.batch(1, 10)
-    print(result)
     result = conn.random()
-    print(result, type(result))
+    print(result)

proxypool/tester.py

@@ -1,61 +0,0 @@
-import asyncio
-import aiohttp
-import time
-import sys
-try:
-    from aiohttp import ClientError
-except:
-    from aiohttp import ClientProxyConnectionError as ProxyConnectionError
-from proxypool.db import RedisClient
-from proxypool.setting import *
-
-
-class Tester(object):
-    def __init__(self):
-        self.redis = RedisClient()
-    
-    async def test_single_proxy(self, proxy):
-        """
-        测试单个代理
-        :param proxy:
-        :return:
-        """
-        conn = aiohttp.TCPConnector(verify_ssl=False)
-        async with aiohttp.ClientSession(connector=conn) as session:
-            try:
-                if isinstance(proxy, bytes):
-                    proxy = proxy.decode('utf-8')
-                real_proxy = 'http://' + proxy
-                print('正在测试', proxy)
-                async with session.get(TEST_URL, proxy=real_proxy, timeout=15, allow_redirects=False) as response:
-                    if response.status in VALID_STATUS_CODES:
-                        self.redis.max(proxy)
-                        print('代理可用', proxy)
-                    else:
-                        self.redis.decrease(proxy)
-                        print('请求响应码不合法 ', response.status, 'IP', proxy)
-            except (ClientError, aiohttp.client_exceptions.ClientConnectorError, asyncio.TimeoutError, AttributeError):
-                self.redis.decrease(proxy)
-                print('代理请求失败', proxy)
-    
-    def run(self):
-        """
-        测试主函数
-        :return:
-        """
-        print('测试器开始运行')
-        try:
-            count = self.redis.count()
-            print('当前剩余', count, '个代理')
-            for i in range(0, count, BATCH_TEST_SIZE):
-                start = i
-                stop = min(i + BATCH_TEST_SIZE, count)
-                print('正在测试第', start + 1, '-', stop, '个代理')
-                test_proxies = self.redis.batch(start, stop)
-                loop = asyncio.get_event_loop()
-                tasks = [self.test_single_proxy(proxy) for proxy in test_proxies]
-                loop.run_until_complete(asyncio.wait(tasks))
-                sys.stdout.flush()
-                time.sleep(5)
-        except Exception as e:
-            print('测试器发生错误', e.args)

proxypool/utils/parse.py

@@ -0,0 +1,13 @@
+import re
+
+def parse_redis_connection_string(connection_string):
+    """
+    parse a redis connection string, for example:
+    redis://[password]@host:port
+    rediss://[password]@host:port
+    :param connection_string:
+    :return:
+    """
+    result = re.match('rediss?:\/\/(.*?)@(.*?):(\d+)', connection_string)
+    return result.group(2), int(result.group(3)), (result.group(1) or None) if result \
+        else ('localhost', 6379, None)

proxypool/utils.py → proxypool/utils/proxy.py

@@ -1,22 +1,6 @@
-import requests
-from requests.exceptions import ConnectionError
+from proxypool.schemas import Proxy
 import re
 
-from proxypool.proxy import Proxy
-
-
-def parse_redis_connection_string(connection_string):
-    """
-    parse a redis connection string, for example:
-    redis://[password]@host:port
-    rediss://[password]@host:port
-    :param connection_string:
-    :return:
-    """
-    result = re.match('rediss?:\/\/(.*?)@(.*?):(\d+)', connection_string)
-    return result.group(2), int(result.group(3)), (result.group(1) or None) if result \
-        else ('localhost', 6379, None)
-
 
 def is_valid_proxy(data):
     """
@@ -33,9 +17,9 @@ def convert_proxy_or_proxies(data):
     :param data:
     :return:
     """
-    print(data)
     if not data:
         return None
+    # if list of proxies
     if isinstance(data, list):
         result = []
         for item in data:

requirements.txt

@@ -1,5 +1,9 @@
-aiohttp>=1.3.3
-Flask>=0.11.1
-redis>=2.10.5
-requests>=2.13.0
-pyquery>=1.2.17
+environs==7.2.0
+Flask==1.0.3
+attrs==19.1.0
+retrying==1.3.3
+aiohttp==3.6.2
+requests==2.22.0
+loguru==0.3.2
+pyquery==1.4.0
+attr==0.3.1

run.py

@@ -1,17 +1,15 @@
 from proxypool.scheduler import Scheduler
-import sys
-import io
+import argparse
 
-sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
-
-
-def main():
-    try:
-        s = Scheduler()
-        s.run()
-    except:
-        main()
 
+parser = argparse.ArgumentParser(description='ProxyPool')
+parser.add_argument('--processor', type=str, help='processor to run')
+args = parser.parse_args()
+print('args', args)
 
 if __name__ == '__main__':
-    main()
+    # if processor set, just run it
+    if args.processor:
+        getattr(Scheduler(), f'run_{args.processor}')()
+    else:
+        Scheduler().run()