| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205 |
- <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
- <html xmlns="http://www.w3.org/1999/xhtml" lang="en">
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-
- <title>proxypool.proxyGetter — proxypool documentation</title>
-
- <link rel="stylesheet" href="../../_static/alabaster.css" type="text/css" />
- <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
-
- <script type="text/javascript">
- var DOCUMENTATION_OPTIONS = {
- URL_ROOT: '../../',
- VERSION: '',
- COLLAPSE_INDEX: false,
- FILE_SUFFIX: '.html',
- HAS_SOURCE: true
- };
- </script>
- <script type="text/javascript" src="../../_static/jquery.js"></script>
- <script type="text/javascript" src="../../_static/underscore.js"></script>
- <script type="text/javascript" src="../../_static/doctools.js"></script>
- <link rel="index" title="Index" href="../../genindex.html" />
- <link rel="search" title="Search" href="../../search.html" />
- <link rel="top" title="proxypool documentation" href="../../index.html" />
- <link rel="up" title="Module code" href="../index.html" />
-
- <link rel="stylesheet" href="../../_static/custom.css" type="text/css" />
-
-
- <meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9" />
- </head>
- <body role="document">
-
- <div class="document">
- <div class="documentwrapper">
- <div class="bodywrapper">
- <div class="body" role="main">
-
- <h1>Source code for proxypool.proxyGetter</h1><div class="highlight"><pre>
- <span></span><span class="sd">"""</span>
- <span class="sd">-------------------------------------------------</span>
- <span class="sd"> File Name: proxyGetter.py</span>
- <span class="sd"> Description: 代理抓取模块,负责与网络的交互。</span>
- <span class="sd"> Author: Liu</span>
- <span class="sd"> Date: 2016/12/9</span>
- <span class="sd">-------------------------------------------------</span>
- <span class="sd">"""</span>
- <span class="kn">import</span> <span class="nn">time</span>
- <span class="kn">from</span> <span class="nn">.utils</span> <span class="k">import</span> <span class="n">get_page</span>
- <div class="viewcode-block" id="ProxyMetaclass"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.ProxyMetaclass">[docs]</a><span class="k">class</span> <span class="nc">ProxyMetaclass</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
- <span class="sd">"""</span>
- <span class="sd"> 爬虫的元类,在FreeProxyGetter类中加入</span>
- <span class="sd"> __CrawlFunc__和__CrawlFuncCount__</span>
- <span class="sd"> 两个参数,分别表示爬虫函数,和爬虫函数的数量。</span>
- <span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__new__</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span>
- <span class="n">count</span> <span class="o">=</span> <span class="mi">0</span>
- <span class="n">attrs</span><span class="p">[</span><span class="s1">'__CrawlFunc__'</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
- <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">attrs</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
- <span class="k">if</span> <span class="s1">'crawl_'</span> <span class="ow">in</span> <span class="n">k</span><span class="p">:</span>
- <span class="n">attrs</span><span class="p">[</span><span class="s1">'__CrawlFunc__'</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">k</span><span class="p">)</span>
- <span class="n">count</span> <span class="o">+=</span> <span class="mi">1</span>
- <span class="n">attrs</span><span class="p">[</span><span class="s1">'__CrawlFuncCount__'</span><span class="p">]</span> <span class="o">=</span> <span class="n">count</span>
- <span class="k">return</span> <span class="nb">type</span><span class="o">.</span><span class="n">__new__</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">attrs</span><span class="p">)</span></div>
- <div class="viewcode-block" id="FreeProxyGetter"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter">[docs]</a><span class="k">class</span> <span class="nc">FreeProxyGetter</span><span class="p">(</span><span class="nb">object</span><span class="p">,</span> <span class="n">metaclass</span><span class="o">=</span><span class="n">ProxyMetaclass</span><span class="p">):</span>
- <span class="sd">"""</span>
- <span class="sd"> 代理爬虫,负责扫描各大代理网站,抓取代理。</span>
- <span class="sd"> 该类有可扩展性,可根据需要自己添加新站点的代理抓取函数,</span>
- <span class="sd"> 但是函数名必须以crawl_开头,返回值必须以"host:port"的形式返回,</span>
- <span class="sd"> 添加器会自动识别并调用此类函数。</span>
- <span class="sd"> """</span>
-
- <div class="viewcode-block" id="FreeProxyGetter.get_raw_proxies"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.get_raw_proxies">[docs]</a> <span class="k">def</span> <span class="nf">get_raw_proxies</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">callback</span><span class="p">,</span> <span class="n">count</span><span class="o">=</span><span class="mi">40</span><span class="p">):</span>
- <span class="n">proxies</span> <span class="o">=</span> <span class="p">[]</span>
- <span class="k">for</span> <span class="n">proxy</span> <span class="ow">in</span> <span class="nb">eval</span><span class="p">(</span><span class="s2">"self.</span><span class="si">{}</span><span class="s2">()"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">callback</span><span class="p">)):</span>
- <span class="n">proxies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">proxy</span><span class="p">)</span>
- <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">proxies</span><span class="p">)</span> <span class="o">>=</span> <span class="n">count</span><span class="p">:</span>
- <span class="k">break</span>
- <span class="k">return</span> <span class="n">proxies</span></div>
- <div class="viewcode-block" id="FreeProxyGetter.crawl_kuaidaili"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.crawl_kuaidaili">[docs]</a> <span class="k">def</span> <span class="nf">crawl_kuaidaili</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">page_count</span><span class="o">=</span><span class="mi">8</span><span class="p">):</span>
- <span class="sd">"""</span>
- <span class="sd"> 抓取快代理网的数据</span>
- <span class="sd"> """</span>
- <span class="n">start_url</span> <span class="o">=</span> <span class="s1">'http://www.kuaidaili.com/proxylist/</span><span class="si">{}</span><span class="s1">/'</span>
- <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="n">start_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">page</span><span class="p">)</span> <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">page_count</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)]</span>
- <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">urls</span><span class="p">:</span>
- <span class="n">soup</span> <span class="o">=</span> <span class="n">get_page</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
-
- <span class="n">proxy_list</span> <span class="o">=</span> <span class="n">soup</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'div'</span><span class="p">,</span> <span class="p">{</span><span class="s1">'id'</span><span class="p">:</span> <span class="s1">'index_free_list'</span><span class="p">})</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'tbody'</span><span class="p">)</span>
- <span class="k">for</span> <span class="n">proxy</span> <span class="ow">in</span> <span class="n">proxy_list</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">'tr'</span><span class="p">):</span>
- <span class="n">ip</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">'td'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span>
- <span class="n">port</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">'td'</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span>
- <span class="k">yield</span> <span class="s1">':'</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">ip</span><span class="p">,</span> <span class="n">port</span><span class="p">])</span></div>
- <div class="viewcode-block" id="FreeProxyGetter.crawl_daili66"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.crawl_daili66">[docs]</a> <span class="k">def</span> <span class="nf">crawl_daili66</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">page_count</span><span class="o">=</span><span class="mi">4</span><span class="p">):</span>
- <span class="sd">"""</span>
- <span class="sd"> 抓取代理66网的数据。</span>
- <span class="sd"> """</span>
- <span class="n">start_url</span> <span class="o">=</span> <span class="s1">'http://www.66ip.cn/</span><span class="si">{}</span><span class="s1">.html'</span>
- <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="n">start_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">page</span><span class="p">)</span> <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">page_count</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)]</span>
- <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">urls</span><span class="p">:</span>
- <span class="n">soup</span> <span class="o">=</span> <span class="n">get_page</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
- <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
- <span class="n">proxy_list</span> <span class="o">=</span> <span class="n">soup</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'table'</span><span class="p">,</span> <span class="p">{</span><span class="s2">"border"</span><span class="p">:</span> <span class="s2">"2px"</span><span class="p">})</span>
- <span class="k">for</span> <span class="n">proxy</span> <span class="ow">in</span> <span class="n">proxy_list</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">'tr'</span><span class="p">)[</span><span class="mi">1</span><span class="p">:]:</span>
- <span class="n">ip</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">'td'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span>
- <span class="n">port</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">'td'</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span>
- <span class="k">yield</span> <span class="s1">':'</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">ip</span><span class="p">,</span> <span class="n">port</span><span class="p">])</span></div>
- <div class="viewcode-block" id="FreeProxyGetter.crawl_xici"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.crawl_xici">[docs]</a> <span class="k">def</span> <span class="nf">crawl_xici</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="sd">"""</span>
- <span class="sd"> 抓取xici代理网的数据。</span>
- <span class="sd"> """</span>
- <span class="n">start_url</span> <span class="o">=</span> <span class="s1">'http://api.xicidaili.com/free2016.txt'</span>
- <span class="n">soup</span> <span class="o">=</span> <span class="n">get_page</span><span class="p">(</span><span class="n">start_url</span><span class="p">)</span>
- <span class="n">proxy_list</span> <span class="o">=</span> <span class="n">soup</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'p'</span><span class="p">)</span>
- <span class="k">return</span> <span class="n">proxy_list</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'</span><span class="se">\r\n</span><span class="s1">'</span><span class="p">)</span></div>
-
- <div class="viewcode-block" id="FreeProxyGetter.crawl_proxy360"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.crawl_proxy360">[docs]</a> <span class="k">def</span> <span class="nf">crawl_proxy360</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="sd">"""</span>
- <span class="sd"> 抓取proxy360网的数据。</span>
- <span class="sd"> """</span>
- <span class="n">start_url</span> <span class="o">=</span> <span class="s1">'http://www.proxy360.cn/default.aspx'</span>
- <span class="n">soup</span> <span class="o">=</span> <span class="n">get_page</span><span class="p">(</span><span class="n">start_url</span><span class="p">)</span>
- <span class="k">for</span> <span class="n">proxy</span> <span class="ow">in</span> <span class="n">soup</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">'div'</span><span class="p">,</span> <span class="p">{</span><span class="s2">"class"</span><span class="p">:</span> <span class="s2">"proxylistitem"</span><span class="p">}):</span>
- <span class="n">item</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">'span'</span><span class="p">,</span> <span class="p">{</span><span class="s2">"class"</span><span class="p">:</span> <span class="s2">"tbBottomLine"</span><span class="p">})</span>
- <span class="n">ip</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\r\n</span><span class="s1">'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">' '</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
- <span class="n">port</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\r\n</span><span class="s1">'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">' '</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
- <span class="k">yield</span> <span class="s1">':'</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">ip</span><span class="p">,</span> <span class="n">port</span><span class="p">])</span></div></div>
-
- <span class="c1"># def crawl_goubanjia(self):</span>
- <span class="c1"># start_url = 'http://www.goubanjia.com/free/gngn/index.shtml'</span>
- <span class="c1"># soup = get_page(start_url)</span>
- <span class="c1"># proxy_list = soup.find('table', {"class": "table"}).find('tbody')</span>
- <span class="c1"># for tr in proxy_list.find_all('tr'):</span>
- <span class="c1"># _proxy = tr.find('td').find_all('span')</span>
- <span class="c1"># proxy = [i.get_text() for i in _proxy]</span>
- <span class="c1"># yield ''.join(proxy)</span>
- <span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
- <span class="kn">import</span> <span class="nn">time</span>
- <span class="n">a</span> <span class="o">=</span> <span class="n">FreeProxyGetter</span><span class="p">()</span>
- <span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">clock</span><span class="p">()</span>
- <span class="nb">print</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">crawl_goubanjia</span><span class="p">()))</span>
-
- <span class="nb">print</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">clock</span><span class="p">()</span><span class="o">-</span><span class="n">start</span><span class="p">)</span>
-
- </pre></div>
- </div>
- </div>
- </div>
- <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
- <div class="sphinxsidebarwrapper"><div class="relations">
- <h3>Related Topics</h3>
- <ul>
- <li><a href="../../index.html">Documentation overview</a><ul>
- <li><a href="../index.html">Module code</a><ul>
- </ul></li>
- </ul></li>
- </ul>
- </div>
- <div id="searchbox" style="display: none" role="search">
- <h3>Quick search</h3>
- <form class="search" action="../../search.html" method="get">
- <div><input type="text" name="q" /></div>
- <div><input type="submit" value="Go" /></div>
- <input type="hidden" name="check_keywords" value="yes" />
- <input type="hidden" name="area" value="default" />
- </form>
- </div>
- <script type="text/javascript">$('#searchbox').show(0);</script>
- </div>
- </div>
- <div class="clearer"></div>
- </div>
- <div class="footer">
- ©2016, Author.
-
- |
- Powered by <a href="http://sphinx-doc.org/">Sphinx 1.4.8</a>
- & <a href="https://github.com/bitprophet/alabaster">Alabaster 0.7.9</a>
-
- </div>
-
-
- </body>
- </html>
|