proxyGetter.html 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205
  1. <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
  2. "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
  3. <html xmlns="http://www.w3.org/1999/xhtml" lang="en">
  4. <head>
  5. <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  6. <title>proxypool.proxyGetter &#8212; proxypool documentation</title>
  7. <link rel="stylesheet" href="../../_static/alabaster.css" type="text/css" />
  8. <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
  9. <script type="text/javascript">
  10. var DOCUMENTATION_OPTIONS = {
  11. URL_ROOT: '../../',
  12. VERSION: '',
  13. COLLAPSE_INDEX: false,
  14. FILE_SUFFIX: '.html',
  15. HAS_SOURCE: true
  16. };
  17. </script>
  18. <script type="text/javascript" src="../../_static/jquery.js"></script>
  19. <script type="text/javascript" src="../../_static/underscore.js"></script>
  20. <script type="text/javascript" src="../../_static/doctools.js"></script>
  21. <link rel="index" title="Index" href="../../genindex.html" />
  22. <link rel="search" title="Search" href="../../search.html" />
  23. <link rel="top" title="proxypool documentation" href="../../index.html" />
  24. <link rel="up" title="Module code" href="../index.html" />
  25. <link rel="stylesheet" href="../../_static/custom.css" type="text/css" />
  26. <meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9" />
  27. </head>
  28. <body role="document">
  29. <div class="document">
  30. <div class="documentwrapper">
  31. <div class="bodywrapper">
  32. <div class="body" role="main">
  33. <h1>Source code for proxypool.proxyGetter</h1><div class="highlight"><pre>
  34. <span></span><span class="sd">&quot;&quot;&quot;</span>
  35. <span class="sd">-------------------------------------------------</span>
  36. <span class="sd"> File Name: proxyGetter.py</span>
  37. <span class="sd"> Description: 代理抓取模块,负责与网络的交互。</span>
  38. <span class="sd"> Author: Liu</span>
  39. <span class="sd"> Date: 2016/12/9</span>
  40. <span class="sd">-------------------------------------------------</span>
  41. <span class="sd">&quot;&quot;&quot;</span>
  42. <span class="kn">import</span> <span class="nn">time</span>
  43. <span class="kn">from</span> <span class="nn">.utils</span> <span class="k">import</span> <span class="n">get_page</span>
  44. <div class="viewcode-block" id="ProxyMetaclass"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.ProxyMetaclass">[docs]</a><span class="k">class</span> <span class="nc">ProxyMetaclass</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
  45. <span class="sd">&quot;&quot;&quot;</span>
  46. <span class="sd"> 爬虫的元类,在FreeProxyGetter类中加入</span>
  47. <span class="sd"> __CrawlFunc__和__CrawlFuncCount__</span>
  48. <span class="sd"> 两个参数,分别表示爬虫函数,和爬虫函数的数量。</span>
  49. <span class="sd"> &quot;&quot;&quot;</span>
  50. <span class="k">def</span> <span class="nf">__new__</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span>
  51. <span class="n">count</span> <span class="o">=</span> <span class="mi">0</span>
  52. <span class="n">attrs</span><span class="p">[</span><span class="s1">&#39;__CrawlFunc__&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
  53. <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">attrs</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
  54. <span class="k">if</span> <span class="s1">&#39;crawl_&#39;</span> <span class="ow">in</span> <span class="n">k</span><span class="p">:</span>
  55. <span class="n">attrs</span><span class="p">[</span><span class="s1">&#39;__CrawlFunc__&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">k</span><span class="p">)</span>
  56. <span class="n">count</span> <span class="o">+=</span> <span class="mi">1</span>
  57. <span class="n">attrs</span><span class="p">[</span><span class="s1">&#39;__CrawlFuncCount__&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">count</span>
  58. <span class="k">return</span> <span class="nb">type</span><span class="o">.</span><span class="n">__new__</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">attrs</span><span class="p">)</span></div>
  59. <div class="viewcode-block" id="FreeProxyGetter"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter">[docs]</a><span class="k">class</span> <span class="nc">FreeProxyGetter</span><span class="p">(</span><span class="nb">object</span><span class="p">,</span> <span class="n">metaclass</span><span class="o">=</span><span class="n">ProxyMetaclass</span><span class="p">):</span>
  60. <span class="sd">&quot;&quot;&quot;</span>
  61. <span class="sd"> 代理爬虫,负责扫描各大代理网站,抓取代理。</span>
  62. <span class="sd"> 该类有可扩展性,可根据需要自己添加新站点的代理抓取函数,</span>
  63. <span class="sd"> 但是函数名必须以crawl_开头,返回值必须以&quot;host:port&quot;的形式返回,</span>
  64. <span class="sd"> 添加器会自动识别并调用此类函数。</span>
  65. <span class="sd"> &quot;&quot;&quot;</span>
  66. <div class="viewcode-block" id="FreeProxyGetter.get_raw_proxies"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.get_raw_proxies">[docs]</a> <span class="k">def</span> <span class="nf">get_raw_proxies</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">callback</span><span class="p">,</span> <span class="n">count</span><span class="o">=</span><span class="mi">40</span><span class="p">):</span>
  67. <span class="n">proxies</span> <span class="o">=</span> <span class="p">[]</span>
  68. <span class="k">for</span> <span class="n">proxy</span> <span class="ow">in</span> <span class="nb">eval</span><span class="p">(</span><span class="s2">&quot;self.</span><span class="si">{}</span><span class="s2">()&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">callback</span><span class="p">)):</span>
  69. <span class="n">proxies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">proxy</span><span class="p">)</span>
  70. <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">proxies</span><span class="p">)</span> <span class="o">&gt;=</span> <span class="n">count</span><span class="p">:</span>
  71. <span class="k">break</span>
  72. <span class="k">return</span> <span class="n">proxies</span></div>
  73. <div class="viewcode-block" id="FreeProxyGetter.crawl_kuaidaili"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.crawl_kuaidaili">[docs]</a> <span class="k">def</span> <span class="nf">crawl_kuaidaili</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">page_count</span><span class="o">=</span><span class="mi">8</span><span class="p">):</span>
  74. <span class="sd">&quot;&quot;&quot;</span>
  75. <span class="sd"> 抓取快代理网的数据</span>
  76. <span class="sd"> &quot;&quot;&quot;</span>
  77. <span class="n">start_url</span> <span class="o">=</span> <span class="s1">&#39;http://www.kuaidaili.com/proxylist/</span><span class="si">{}</span><span class="s1">/&#39;</span>
  78. <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="n">start_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">page</span><span class="p">)</span> <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">page_count</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)]</span>
  79. <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">urls</span><span class="p">:</span>
  80. <span class="n">soup</span> <span class="o">=</span> <span class="n">get_page</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
  81. <span class="n">proxy_list</span> <span class="o">=</span> <span class="n">soup</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;div&#39;</span><span class="p">,</span> <span class="p">{</span><span class="s1">&#39;id&#39;</span><span class="p">:</span> <span class="s1">&#39;index_free_list&#39;</span><span class="p">})</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;tbody&#39;</span><span class="p">)</span>
  82. <span class="k">for</span> <span class="n">proxy</span> <span class="ow">in</span> <span class="n">proxy_list</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">&#39;tr&#39;</span><span class="p">):</span>
  83. <span class="n">ip</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">&#39;td&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span>
  84. <span class="n">port</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">&#39;td&#39;</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span>
  85. <span class="k">yield</span> <span class="s1">&#39;:&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">ip</span><span class="p">,</span> <span class="n">port</span><span class="p">])</span></div>
  86. <div class="viewcode-block" id="FreeProxyGetter.crawl_daili66"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.crawl_daili66">[docs]</a> <span class="k">def</span> <span class="nf">crawl_daili66</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">page_count</span><span class="o">=</span><span class="mi">4</span><span class="p">):</span>
  87. <span class="sd">&quot;&quot;&quot;</span>
  88. <span class="sd"> 抓取代理66网的数据。</span>
  89. <span class="sd"> &quot;&quot;&quot;</span>
  90. <span class="n">start_url</span> <span class="o">=</span> <span class="s1">&#39;http://www.66ip.cn/</span><span class="si">{}</span><span class="s1">.html&#39;</span>
  91. <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span><span class="n">start_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">page</span><span class="p">)</span> <span class="k">for</span> <span class="n">page</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">page_count</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)]</span>
  92. <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">urls</span><span class="p">:</span>
  93. <span class="n">soup</span> <span class="o">=</span> <span class="n">get_page</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
  94. <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
  95. <span class="n">proxy_list</span> <span class="o">=</span> <span class="n">soup</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;table&#39;</span><span class="p">,</span> <span class="p">{</span><span class="s2">&quot;border&quot;</span><span class="p">:</span> <span class="s2">&quot;2px&quot;</span><span class="p">})</span>
  96. <span class="k">for</span> <span class="n">proxy</span> <span class="ow">in</span> <span class="n">proxy_list</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">&#39;tr&#39;</span><span class="p">)[</span><span class="mi">1</span><span class="p">:]:</span>
  97. <span class="n">ip</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">&#39;td&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span>
  98. <span class="n">port</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">&#39;td&#39;</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span>
  99. <span class="k">yield</span> <span class="s1">&#39;:&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">ip</span><span class="p">,</span> <span class="n">port</span><span class="p">])</span></div>
  100. <div class="viewcode-block" id="FreeProxyGetter.crawl_xici"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.crawl_xici">[docs]</a> <span class="k">def</span> <span class="nf">crawl_xici</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
  101. <span class="sd">&quot;&quot;&quot;</span>
  102. <span class="sd"> 抓取xici代理网的数据。</span>
  103. <span class="sd"> &quot;&quot;&quot;</span>
  104. <span class="n">start_url</span> <span class="o">=</span> <span class="s1">&#39;http://api.xicidaili.com/free2016.txt&#39;</span>
  105. <span class="n">soup</span> <span class="o">=</span> <span class="n">get_page</span><span class="p">(</span><span class="n">start_url</span><span class="p">)</span>
  106. <span class="n">proxy_list</span> <span class="o">=</span> <span class="n">soup</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">&#39;p&#39;</span><span class="p">)</span>
  107. <span class="k">return</span> <span class="n">proxy_list</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\r\n</span><span class="s1">&#39;</span><span class="p">)</span></div>
  108. <div class="viewcode-block" id="FreeProxyGetter.crawl_proxy360"><a class="viewcode-back" href="../../proxypool.html#proxypool.proxyGetter.FreeProxyGetter.crawl_proxy360">[docs]</a> <span class="k">def</span> <span class="nf">crawl_proxy360</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
  109. <span class="sd">&quot;&quot;&quot;</span>
  110. <span class="sd"> 抓取proxy360网的数据。</span>
  111. <span class="sd"> &quot;&quot;&quot;</span>
  112. <span class="n">start_url</span> <span class="o">=</span> <span class="s1">&#39;http://www.proxy360.cn/default.aspx&#39;</span>
  113. <span class="n">soup</span> <span class="o">=</span> <span class="n">get_page</span><span class="p">(</span><span class="n">start_url</span><span class="p">)</span>
  114. <span class="k">for</span> <span class="n">proxy</span> <span class="ow">in</span> <span class="n">soup</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">&#39;div&#39;</span><span class="p">,</span> <span class="p">{</span><span class="s2">&quot;class&quot;</span><span class="p">:</span> <span class="s2">&quot;proxylistitem&quot;</span><span class="p">}):</span>
  115. <span class="n">item</span> <span class="o">=</span> <span class="n">proxy</span><span class="o">.</span><span class="n">find_all</span><span class="p">(</span><span class="s1">&#39;span&#39;</span><span class="p">,</span> <span class="p">{</span><span class="s2">&quot;class&quot;</span><span class="p">:</span> <span class="s2">&quot;tbBottomLine&quot;</span><span class="p">})</span>
  116. <span class="n">ip</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\r\n</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
  117. <span class="n">port</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">get_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;</span><span class="se">\r\n</span><span class="s1">&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
  118. <span class="k">yield</span> <span class="s1">&#39;:&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">ip</span><span class="p">,</span> <span class="n">port</span><span class="p">])</span></div></div>
  119. <span class="c1"># def crawl_goubanjia(self):</span>
  120. <span class="c1"># start_url = &#39;http://www.goubanjia.com/free/gngn/index.shtml&#39;</span>
  121. <span class="c1"># soup = get_page(start_url)</span>
  122. <span class="c1"># proxy_list = soup.find(&#39;table&#39;, {&quot;class&quot;: &quot;table&quot;}).find(&#39;tbody&#39;)</span>
  123. <span class="c1"># for tr in proxy_list.find_all(&#39;tr&#39;):</span>
  124. <span class="c1"># _proxy = tr.find(&#39;td&#39;).find_all(&#39;span&#39;)</span>
  125. <span class="c1"># proxy = [i.get_text() for i in _proxy]</span>
  126. <span class="c1"># yield &#39;&#39;.join(proxy)</span>
  127. <span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
  128. <span class="kn">import</span> <span class="nn">time</span>
  129. <span class="n">a</span> <span class="o">=</span> <span class="n">FreeProxyGetter</span><span class="p">()</span>
  130. <span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">clock</span><span class="p">()</span>
  131. <span class="nb">print</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">crawl_goubanjia</span><span class="p">()))</span>
  132. <span class="nb">print</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">clock</span><span class="p">()</span><span class="o">-</span><span class="n">start</span><span class="p">)</span>
  133. </pre></div>
  134. </div>
  135. </div>
  136. </div>
  137. <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
  138. <div class="sphinxsidebarwrapper"><div class="relations">
  139. <h3>Related Topics</h3>
  140. <ul>
  141. <li><a href="../../index.html">Documentation overview</a><ul>
  142. <li><a href="../index.html">Module code</a><ul>
  143. </ul></li>
  144. </ul></li>
  145. </ul>
  146. </div>
  147. <div id="searchbox" style="display: none" role="search">
  148. <h3>Quick search</h3>
  149. <form class="search" action="../../search.html" method="get">
  150. <div><input type="text" name="q" /></div>
  151. <div><input type="submit" value="Go" /></div>
  152. <input type="hidden" name="check_keywords" value="yes" />
  153. <input type="hidden" name="area" value="default" />
  154. </form>
  155. </div>
  156. <script type="text/javascript">$('#searchbox').show(0);</script>
  157. </div>
  158. </div>
  159. <div class="clearer"></div>
  160. </div>
  161. <div class="footer">
  162. &copy;2016, Author.
  163. |
  164. Powered by <a href="http://sphinx-doc.org/">Sphinx 1.4.8</a>
  165. &amp; <a href="https://github.com/bitprophet/alabaster">Alabaster 0.7.9</a>
  166. </div>
  167. </body>
  168. </html>