ProxyPool/proxypool/crawlers/public/hidemy.py

import re

from pyquery import PyQuery as pq
from proxypool.schemas.proxy import Proxy
from proxypool.crawlers.base import BaseCrawler
from loguru import logger


class HidemyCrawler(BaseCrawler):

    urls  = [
            'https://hidemy.name/en/proxy-list/?type=h&anon=4#list',
            'https://hidemy.name/en/proxy-list/?type=h&anon=4&start=64#list',
            'https://hidemy.name/en/proxy-list/?type=h&anon=4&start=128#list',
            'https://hidemy.name/en/proxy-list/?type=h&anon=4&start=192#list',
        ]

    @logger.catch
    def crawl(self):
        """
        crawl main method
        """
        for url in self.urls:
            logger.info(f'fetching {url}')
            html = self.fetch(url)
            for proxy in self.parse(html):
                logger.info(f'fetched proxy {proxy.string()} from {url}')
                yield proxy

    def parse(self, html):
        """
        parse html file to get proxies
        :return:
        """
        print(html)
        proxies = re.findall(
            r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td><td>(\d+)</td>',
            html)
        for proxy in proxies:
            yield Proxy(host=proxy[0], port=proxy[1])


if __name__ == '__main__':
    crawler = HidemyCrawler()
    for proxy in crawler.crawl():
        print(proxy)