update struct

.idea/workspace.xml

@@ -2,8 +2,14 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="24d4e717-d5fe-45ea-9d4b-5aab93817d90" name="Default" comment="">
-      <change type="MOVED" beforePath="$PROJECT_DIR$/proxypool/proxyGetter.py" afterPath="$PROJECT_DIR$/proxypool/getter.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/proxypool/__init__.py" afterPath="$PROJECT_DIR$/proxypool/__init__.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/proxypool/api.py" afterPath="$PROJECT_DIR$/proxypool/api.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/proxypool/db.py" afterPath="$PROJECT_DIR$/proxypool/db.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/proxypool/error.py" afterPath="$PROJECT_DIR$/proxypool/error.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/proxypool/getter.py" afterPath="$PROJECT_DIR$/proxypool/getter.py" />
       <change type="MODIFICATION" beforePath="$PROJECT_DIR$/proxypool/schedule.py" afterPath="$PROJECT_DIR$/proxypool/schedule.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/proxypool/setting.py" afterPath="$PROJECT_DIR$/proxypool/setting.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/proxypool/utils.py" afterPath="$PROJECT_DIR$/proxypool/utils.py" />
     </list>
     <ignored path="ProxyPool.iws" />
     <ignored path=".idea/workspace.xml" />
@@ -18,6 +24,7 @@
   <component name="ChangesViewManager" flattened_view="true" show_ignored="false" />
   <component name="CoverageDataManager">
     <SUITE FILE_PATH="coverage/ProxyPool$example.coverage" NAME="example Coverage Results" MODIFIED="1487830200820" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/examples" />
+    <SUITE FILE_PATH="coverage/ProxyPool$db.coverage" NAME="db Coverage Results" MODIFIED="1487855837836" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/proxypool" />
     <SUITE FILE_PATH="coverage/ProxyPool$testaio.coverage" NAME="testaio Coverage Results" MODIFIED="1487831286992" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/examples" />
     <SUITE FILE_PATH="coverage/ProxyPool$testaio2.coverage" NAME="testaio2 Coverage Results" MODIFIED="1487834393115" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/examples" />
   </component>
@@ -45,8 +52,8 @@
       <file leaf-file-name="utils.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/proxypool/utils.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="114">
-              <caret line="19" column="8" selection-start-line="19" selection-start-column="8" selection-end-line="19" selection-end-column="8" />
+            <state relative-caret-position="-338">
+              <caret line="10" column="0" selection-start-line="10" selection-start-column="0" selection-end-line="10" selection-end-column="0" />
               <folding>
                 <element signature="e#0#15#0" expanded="true" />
               </folding>
@@ -57,10 +64,10 @@
       <file leaf-file-name="getter.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/proxypool/getter.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="187">
-              <caret line="49" column="33" selection-start-line="49" selection-start-column="33" selection-end-line="49" selection-end-column="33" />
+            <state relative-caret-position="-19">
+              <caret line="29" column="4" selection-start-line="29" selection-start-column="4" selection-end-line="29" selection-end-column="4" />
               <folding>
-                <element signature="e#397#408#0" expanded="true" />
+                <element signature="e#0#27#0" expanded="true" />
               </folding>
             </state>
           </provider>
@@ -69,32 +76,40 @@
       <file leaf-file-name="error.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/proxypool/error.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-19">
-              <caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+            <state relative-caret-position="57">
+              <caret line="3" column="11" selection-start-line="3" selection-start-column="11" selection-end-line="3" selection-end-column="11" />
               <folding />
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="schedule.py" pinned="false" current-in-tab="true">
+      <file leaf-file-name="schedule.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/proxypool/schedule.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="377">
-              <caret line="94" column="35" selection-start-line="94" selection-start-column="35" selection-end-line="94" selection-end-column="35" />
-              <folding>
-                <element signature="e#304#315#0" expanded="true" />
-              </folding>
+            <state relative-caret-position="133">
+              <caret line="7" column="14" selection-start-line="7" selection-start-column="14" selection-end-line="7" selection-end-column="14" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file leaf-file-name="__init__.py" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/proxypool/__init__.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="0">
+              <caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+              <folding />
             </state>
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="db.py" pinned="false" current-in-tab="false">
+      <file leaf-file-name="db.py" pinned="false" current-in-tab="true">
         <entry file="file://$PROJECT_DIR$/proxypool/db.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="-98">
-              <caret line="34" column="33" selection-start-line="34" selection-start-column="33" selection-end-line="34" selection-end-column="33" />
+            <state relative-caret-position="1045">
+              <caret line="55" column="26" selection-start-line="55" selection-start-column="26" selection-end-line="55" selection-end-column="26" />
               <folding>
-                <element signature="e#247#259#0" expanded="true" />
+                <element signature="e#0#12#0" expanded="true" />
               </folding>
             </state>
           </provider>
@@ -103,10 +118,10 @@
       <file leaf-file-name="api.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/proxypool/api.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="42">
-              <caret line="28" column="25" selection-start-line="28" selection-start-column="25" selection-end-line="28" selection-end-column="25" />
+            <state relative-caret-position="0">
+              <caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
               <folding>
-                <element signature="e#436#462#0" expanded="true" />
+                <element signature="e#0#26#0" expanded="true" />
               </folding>
             </state>
           </provider>
@@ -115,8 +130,8 @@
       <file leaf-file-name="setting.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/proxypool/setting.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="247">
-              <caret line="13" column="0" selection-start-line="13" selection-start-column="0" selection-end-line="13" selection-end-column="0" />
+            <state relative-caret-position="0">
+              <caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
               <folding />
             </state>
           </provider>
@@ -142,14 +157,18 @@
       <list>
         <option value="$PROJECT_DIR$/examples/testaio.py" />
         <option value="$PROJECT_DIR$/examples/testaio2.py" />
-        <option value="$PROJECT_DIR$/proxypool/utils.py" />
-        <option value="$PROJECT_DIR$/proxypool/setting.py" />
         <option value="$PROJECT_DIR$/README.md" />
         <option value="$PROJECT_DIR$/docs/_build/doctrees/environment.pickle" />
         <option value="$PROJECT_DIR$/.travis.yml" />
         <option value="$PROJECT_DIR$/proxypool/proxyGetter.py" />
-        <option value="$PROJECT_DIR$/proxypool/getter.py" />
+        <option value="$PROJECT_DIR$/proxypool/error.py" />
+        <option value="$PROJECT_DIR$/proxypool/setting.py" />
+        <option value="$PROJECT_DIR$/proxypool/__init__.py" />
         <option value="$PROJECT_DIR$/proxypool/schedule.py" />
+        <option value="$PROJECT_DIR$/proxypool/api.py" />
+        <option value="$PROJECT_DIR$/proxypool/utils.py" />
+        <option value="$PROJECT_DIR$/proxypool/db.py" />
+        <option value="$PROJECT_DIR$/proxypool/getter.py" />
       </list>
     </option>
   </component>
@@ -264,7 +283,7 @@
     <property name="js-jscs-nodeInterpreter" value="/usr/local/bin/node" />
     <property name="FullScreen" value="true" />
   </component>
-  <component name="RunManager" selected="Python.testaio2">
+  <component name="RunManager" selected="Python.db">
     <configuration default="false" name="example" type="PythonConfigurationType" factoryName="Python" temporary="true">
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="PARENT_ENVS" value="true" />
@@ -319,6 +338,24 @@
       <option name="SHOW_COMMAND_LINE" value="false" />
       <method />
     </configuration>
+    <configuration default="false" name="db" type="PythonConfigurationType" factoryName="Python" temporary="true">
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/proxypool" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <module name="ProxyPool" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/proxypool/db.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <method />
+    </configuration>
     <configuration default="true" type="BashConfigurationType" factoryName="Bash">
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="INTERPRETER_PATH" value="/bin/bash" />
@@ -532,16 +569,18 @@
       <option name="USE_KEYWORD" value="false" />
       <method />
     </configuration>
-    <list size="3">
+    <list size="4">
       <item index="0" class="java.lang.String" itemvalue="Python.example" />
       <item index="1" class="java.lang.String" itemvalue="Python.testaio" />
       <item index="2" class="java.lang.String" itemvalue="Python.testaio2" />
+      <item index="3" class="java.lang.String" itemvalue="Python.db" />
     </list>
     <recent_temporary>
-      <list size="3">
-        <item index="0" class="java.lang.String" itemvalue="Python.testaio2" />
-        <item index="1" class="java.lang.String" itemvalue="Python.testaio" />
-        <item index="2" class="java.lang.String" itemvalue="Python.example" />
+      <list size="4">
+        <item index="0" class="java.lang.String" itemvalue="Python.db" />
+        <item index="1" class="java.lang.String" itemvalue="Python.testaio2" />
+        <item index="2" class="java.lang.String" itemvalue="Python.testaio" />
+        <item index="3" class="java.lang.String" itemvalue="Python.example" />
       </list>
     </recent_temporary>
   </component>
@@ -675,36 +714,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/examples/example.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="285">
-          <caret line="15" column="16" selection-start-line="15" selection-start-column="16" selection-end-line="15" selection-end-column="16" />
-          <folding>
-            <element signature="e#0#9#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/proxypool/api.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="42">
-          <caret line="28" column="25" selection-start-line="28" selection-start-column="25" selection-end-line="28" selection-end-column="25" />
-          <folding>
-            <element signature="e#436#462#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
-    <entry file="file://$PROJECT_DIR$/proxypool/utils.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="114">
-          <caret line="19" column="8" selection-start-line="19" selection-start-column="8" selection-end-line="19" selection-end-column="8" />
-          <folding>
-            <element signature="e#0#15#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/docs/_build/doctrees/environment.pickle">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="203">
@@ -732,48 +741,84 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/proxypool/setting.py">
+    <entry file="file://$PROJECT_DIR$/proxypool/__init__.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="247">
-          <caret line="13" column="0" selection-start-line="13" selection-start-column="0" selection-end-line="13" selection-end-column="0" />
+        <state relative-caret-position="0">
+          <caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/proxypool/db.py">
+    <entry file="file://$PROJECT_DIR$/proxypool/utils.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-98">
-          <caret line="34" column="33" selection-start-line="34" selection-start-column="33" selection-end-line="34" selection-end-column="33" />
+        <state relative-caret-position="-338">
+          <caret line="10" column="0" selection-start-line="10" selection-start-column="0" selection-end-line="10" selection-end-column="0" />
           <folding>
-            <element signature="e#247#259#0" expanded="true" />
+            <element signature="e#0#15#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/proxypool/error.py">
+    <entry file="file://$PROJECT_DIR$/proxypool/api.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="-19">
+        <state relative-caret-position="0">
           <caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
-          <folding />
+          <folding>
+            <element signature="e#0#26#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/examples/example.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="285">
+          <caret line="15" column="16" selection-start-line="15" selection-start-column="16" selection-end-line="15" selection-end-column="16" />
+          <folding>
+            <element signature="e#0#9#0" expanded="true" />
+          </folding>
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/proxypool/getter.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="187">
-          <caret line="49" column="33" selection-start-line="49" selection-start-column="33" selection-end-line="49" selection-end-column="33" />
+        <state relative-caret-position="-19">
+          <caret line="29" column="4" selection-start-line="29" selection-start-column="4" selection-end-line="29" selection-end-column="4" />
           <folding>
-            <element signature="e#397#408#0" expanded="true" />
+            <element signature="e#0#27#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
     <entry file="file://$PROJECT_DIR$/proxypool/schedule.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="377">
-          <caret line="94" column="35" selection-start-line="94" selection-start-column="35" selection-end-line="94" selection-end-column="35" />
+        <state relative-caret-position="133">
+          <caret line="7" column="14" selection-start-line="7" selection-start-column="14" selection-end-line="7" selection-end-column="14" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/proxypool/setting.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/proxypool/error.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="57">
+          <caret line="3" column="11" selection-start-line="3" selection-start-column="11" selection-end-line="3" selection-end-column="11" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/proxypool/db.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1045">
+          <caret line="55" column="26" selection-start-line="55" selection-start-column="26" selection-end-line="55" selection-end-column="26" />
           <folding>
-            <element signature="e#304#315#0" expanded="true" />
+            <element signature="e#0#12#0" expanded="true" />
           </folding>
         </state>
       </provider>

proxypool/__init__.py

@@ -1 +0,0 @@
-__author__ = 'WiseDoge'

proxypool/api.py

@@ -1,16 +1,3 @@
-"""
--------------------------------------------------
-    File Name:     api.py
-    Description:   API模块，运行后打开浏览器，访问
-                   http://127.0.0.1:5000/进入主页。
-                   访问 http://127.0.0.1:5000/get 
-                   从代理池中获取一个代理。 
-                   访问 http://127.0.0.1:5000/count
-                   获取代理池中可用代理的总数。
-    Author:        Liu
-    Date:          2016/12/9
--------------------------------------------------
-"""
 from flask import Flask, g
 
 from .db import RedisClient

proxypool/db.py

@@ -1,16 +1,6 @@
-"""
--------------------------------------------------
-    File Name:     db.py
-    Description:   数据库操作模块，负责对象与底层数据库
-                   的交互。
-    Author:        Liu
-    Date:          2016/12/9
--------------------------------------------------
-"""
 import redis
-
-from .error import PoolEmptyError
-from .setting import HOST, PORT
+from proxypool.error import PoolEmptyError
+from proxypool.setting import HOST, PORT
 
 
 class RedisClient(object):
@@ -19,12 +9,12 @@ class RedisClient(object):
     """
 
     def __init__(self, host=HOST, port=PORT):
-        self.__db = redis.Redis(host, port)
+        self._db = redis.Redis(host, port)
 
     def get(self, count=1):
         """从Pool中获取一定量数据。"""
-        proxies = self.__db.lrange("proxies", 0, count - 1)
-        self.__db.ltrim("proxies", count, -1)
+        proxies = self._db.lrange("proxies", 0, count - 1)
+        self._db.ltrim("proxies", count, -1)
         return proxies
 
     def put(self, proxy):
@@ -32,8 +22,8 @@ class RedisClient(object):
         用Redis的set容器来负责去重，如果proxy能被压入proxy_set，
         就将其放入proxy pool中，否则不压入。
         """
-        if self.__db.sadd("proxy_set", proxy):
-            self.__db.rpush("proxies", proxy)
+        if self._db.sadd("set", proxy):
+            self._db.rpush("proxies", proxy)
         else:
             pass
 
@@ -47,7 +37,7 @@ class RedisClient(object):
         """弹出一个可用代理。
         """
         try:
-            return self.__db.blpop("proxies", 30)[1].decode('utf-8')
+            return self._db.blpop("proxies", 30)[1].decode('utf-8')
         except:
             raise PoolEmptyError
 
@@ -55,9 +45,14 @@ class RedisClient(object):
     def queue_len(self):
         """获取proxy pool的大小。
         """
-        return self.__db.llen("proxies")
+        return self._db.llen("proxies")
 
     def flush(self):
         """刷新Redis中的全部内容，测试用。
         """
-        self.__db.flushall()
+        self._db.flushall()
+
+
+if __name__ == '__main__':
+    conn = RedisClient()
+    print(conn.get(20))

proxypool/error.py

@@ -1,13 +1,3 @@
-"""
--------------------------------------------------
-    File Name:     error.py
-    Description:   异常模块
-    Author:        Liu
-    Date:          2016/12/9
--------------------------------------------------
-"""
-
-
 class ResourceDepletionError(Exception):
     """
     资源枯竭异常，如果从所有抓取网站都抓不到可用的代理资源，

proxypool/getter.py

@@ -1,18 +1,3 @@
-"""
--------------------------------------------------
-    File Name:     proxyGetter.py
-    Description:   代理抓取模块，负责与网络的交互。
-                   注意，代理网站的HTML结构可能会时常的更新，
-                   会导致本文件下的抓取函数失效，所以，在运行
-                   代理池之前，需要更新一下FreeProxyGetter类
-                   中以crawl_开头的方法。
-    Author:        Liu
-    Date:          2016/12/9
--------------------------------------------------
-"""
-
-import time
-
 from .utils import get_page
 from pyquery import PyQuery as pq
 
@@ -43,7 +28,7 @@ class FreeProxyGetter(object, metaclass=ProxyMetaclass):
     添加器会自动识别并调用此类函数。
     """
     
-    def get_raw_proxies(self, callback, count=40):
+    def get_raw_proxies(self, callback):
         proxies = []
         print('Callback', callback)
         for proxy in eval("self.{}()".format(callback)):

proxypool/schedule.py

@@ -1,21 +1,11 @@
-"""
--------------------------------------------------
-    File Name:     schedule.py
-    Description:   调度器模块，
-                   包含ValidityTester，PoolAdder，
-                   Schedule三个类，负责维护代理池。
-    Author:        Liu
-    Date:          2016/12/9
--------------------------------------------------
-"""
 import time
 from multiprocessing import Process
 import asyncio
 import aiohttp
-from .db import RedisClient
-from .error import ResourceDepletionError
-from .getter import FreeProxyGetter
-from .setting import *
+from proxypool.db import RedisClient
+from proxypool.error import ResourceDepletionError
+from proxypool.getter import FreeProxyGetter
+from proxypool.setting import *
 
 
 class ValidityTester(object):
@@ -23,20 +13,22 @@ class ValidityTester(object):
     检验器，负责对未知的代理进行异步检测。
     """
     # 用百度的首页来检验
-    test_api = 'http://www.baidu.com'
+    test_api = TEST_API
 
     def __init__(self):
         self._raw_proxies = None
         self._usable_proxies = []
 
     def set_raw_proxies(self, proxies):
-        """设置待检测的代理。
+        """
+        设置待检测的代理。
         """
         self._raw_proxies = proxies
         self._usable_proxies = []
 
     async def test_single_proxy(self, proxy):
-        """检测单个代理，如果可用，则将其加入_usable_proxies
+        """
+        检测单个代理，如果可用，则将其加入_usable_proxies
         """
         async with aiohttp.ClientSession() as session:
             try:
@@ -44,13 +36,14 @@ class ValidityTester(object):
                 print('Testing', real_proxy)
                 async with session.get(self.test_api, proxy=real_proxy, timeout=15) as response:
                     await response
-                    print('Response from', proxy)
                     self._usable_proxies.append(proxy)
+                    print('Valid proxy', proxy)
             except Exception:
-                pass
+                print('Invalid proxy', proxy)
 
     def test(self):
-        """异步检测_raw_proxies中的全部代理。
+        """
+        异步检测_raw_proxies中的全部代理。
         """
         print('ValidityTester is working')
         loop = asyncio.get_event_loop()
@@ -92,9 +85,6 @@ class PoolAdder(object):
             for callback_label in range(self._crawler.__CrawlFuncCount__):
                 callback = self._crawler.__CrawlFunc__[callback_label]
                 raw_proxies = self._crawler.get_raw_proxies(callback)
-                self._tester.set_raw_proxies(raw_proxies)
-                self._tester.test()
-                self._conn.put_many(self._tester.get_usable_proxies())
                 proxy_count += len(raw_proxies)
             if proxy_count == 0:
                 raise ResourceDepletionError

proxypool/setting.py

@@ -1,12 +1,3 @@
-"""
--------------------------------------------------
-    File Name:     setting.py
-    Description:   设置模块，包含了一些常量。
-    Author:        Liu
-    Date:          2016/12/9
--------------------------------------------------
-"""
-
 # Redis Host
 HOST = 'localhost'
 # Redis PORT
@@ -18,3 +9,5 @@ POOL_UPPER_THRESHOLD = 200
 
 VALID_CHECK_CYCLE = 600
 POOL_LEN_CHECK_CYCLE = 20
+
+TEST_API='http://www.baidu.com'

proxypool/utils.py

@@ -1,14 +1,10 @@
 import requests
-import lxml
 import asyncio
-import time
 import aiohttp
-from bs4 import BeautifulSoup
 from requests.exceptions import ConnectionError
 
 base_headers = {
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \
-    (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
     'Accept-Encoding': 'gzip, deflate, sdch',
     'Accept-Language': 'zh-CN,zh;q=0.8'
 }
@@ -16,7 +12,7 @@ base_headers = {
 
 def get_page(url, options={}):
     headers = dict(base_headers, **options)
-    print('Getting', url, headers)
+    print('Getting', url)
     try:
         r = requests.get(url, headers=headers)
         print('Getting result', url, r.status_code)