update kudaidaili

deployment.yml

@@ -39,26 +39,6 @@ items:
         io.kompose.service: proxypool
     status:
       loadBalancer: {}
-  - apiVersion: v1
-    kind: Service
-    metadata:
-      annotations:
-        kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml
-        kompose.version: 1.20.0 ()
-      creationTimestamp: null
-      labels:
-        io.kompose.service: redis
-      name: redis
-      namespace: proxypool
-    spec:
-      ports:
-        - name: "6379"
-          port: 6379
-          targetPort: 6379
-      selector:
-        io.kompose.service: redis
-    status:
-      loadBalancer: {}
   - apiVersion: extensions/v1beta1
     kind: Deployment
     metadata:
@@ -85,10 +65,11 @@ items:
         spec:
           containers:
             - env:
-                - name: REDIS_HOST
-                  value: redis
-                - name: REDIS_PORT
-                  value: '6379'
+                - name: REDIS_CONNECTION_STRING
+                  valueFrom:
+                    secretKeyRef:
+                      name: redis
+                      key: connection_string
               image: germey/proxypool:${TAG}
               name: proxypool
               ports:
@@ -103,39 +84,5 @@ items:
               persistentVolumeClaim:
                 claimName: proxypool
     status: {}
-  - apiVersion: extensions/v1beta1
-    kind: Deployment
-    metadata:
-      annotations:
-        kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml
-        kompose.version: 1.20.0 ()
-      creationTimestamp: null
-      labels:
-        io.kompose.service: redis
-      name: redis
-      namespace: proxypool
-    spec:
-      replicas: 1
-      revisionHistoryLimit: 0
-      strategy: {}
-      template:
-        metadata:
-          annotations:
-            kompose.cmd: kompose convert -f docker-compose.yml -o deployment.yml
-            kompose.version: 1.20.0 ()
-          creationTimestamp: null
-          labels:
-            io.kompose.service: redis
-        spec:
-          containers:
-            - args:
-                - redis-server
-              image: redis:alpine
-              name: redis
-              ports:
-                - containerPort: 6379
-              resources: {}
-          restartPolicy: Always
-    status: {}
 kind: List
 metadata: {}

proxypool/crawlers/private/.gitignore

@@ -0,0 +1,2 @@
+*
+!__init__.py

proxypool/crawlers/public/kuaidaili.py

@@ -0,0 +1,32 @@
+from proxypool.crawlers.base import BaseCrawler
+from proxypool.schemas.proxy import Proxy
+import re
+from pyquery import PyQuery as pq
+
+
+BASE_URL = 'https://www.kuaidaili.com/free/inha/{page}/'
+
+
+class KuaidailiCrawler(BaseCrawler):
+    """
+    kuaidaili crawler, https://www.kuaidaili.com/
+    """
+    urls = [BASE_URL.format(page=page) for page in range(1, 100)]
+    
+    def parse(self, html):
+        """
+        parse html file to get proxies
+        :return:
+        """
+        doc = pq(html)
+        for item in doc('table tr').items():
+            td_ip = item.find('td[data-title="IP"]').text()
+            td_port = item.find('td[data-title="PORT"]').text()
+            if td_ip and td_port:
+                yield Proxy(host=td_ip, port=td_port)
+
+
+if __name__ == '__main__':
+    crawler = KuaidailiCrawler()
+    for proxy in crawler.crawl():
+        print(proxy)