update proxy pool

examples/proxytest.py

@@ -0,0 +1,15 @@
+import requests
+from proxypool.setting import TEST_URL
+
+proxy = '49.86.62.22:808'
+
+proxies = {
+    'http': 'http://' + proxy,
+    'https': 'https://' + proxy,
+}
+
+print(TEST_URL)
+response = requests.get(TEST_URL, proxies=proxies, verify=False)
+if response.status_code == 200:
+    print('Successfully')
+    print(response.text)

proxypool/db.py

@@ -33,7 +33,7 @@ class RedisClient(object):
         :param score:
         :return:
         """
-        self.db.zadd(REDIS_KEY, score, proxy)
+        return self.db.zadd(REDIS_KEY, score, proxy)
     
     def random(self):
         """
@@ -74,7 +74,7 @@ class RedisClient(object):
         :param proxy:
         :return:
         """
-        self.db.zadd(REDIS_KEY, MAX_SCORE, proxy)
+        return self.db.zadd(REDIS_KEY, MAX_SCORE, proxy)
     
     def count(self):
         """
@@ -90,7 +90,7 @@ class RedisClient(object):
         """
         all = self.db.zrangebyscore(REDIS_KEY, MIN_SCORE, MAX_SCORE)
         return [item.decode('utf-8') for item in all]
-
+    
 
 if __name__ == '__main__':
     conn = RedisClient()

proxypool/importer.py

@@ -0,0 +1,24 @@
+import requests
+
+from proxypool.db import RedisClient
+
+conn = RedisClient()
+
+
+def set(proxy):
+    result = conn.add(proxy)
+    print(proxy)
+    print('录入成功' if result else '录入失败')
+
+
+def scan():
+    print('请输入代理, 输入exit退出读入')
+    while True:
+        account = input()
+        if account == 'exit':
+            break
+        set(account)
+
+
+if __name__ == '__main__':
+    scan()

proxypool/scheduler.py

@@ -6,8 +6,8 @@ from proxypool.tester import Tester
 from proxypool.db import RedisClient
 from proxypool.setting import *
 
-class Scheduler(object):
-    def schedule_tester(cycle=TESTER_CYCLE):
+class Scheduler():
+    def schedule_tester(self, cycle=TESTER_CYCLE):
         """
         定时测试代理
         """
@@ -26,7 +26,7 @@ class Scheduler(object):
             tester.run()
             time.sleep(cycle)
     
-    def schedule_getter(cycle=GETTER_CYCLE):
+    def schedule_getter(self, cycle=GETTER_CYCLE):
         """
         定时获取代理
         """

proxypool/setting.py

@@ -16,12 +16,12 @@ VALID_STATUS_CODES = [200]
 POOL_UPPER_THRESHOLD = 10000
 
 # 检查周期
-TESTER_CYCLE = 30
+TESTER_CYCLE = 20
 # 获取周期
 GETTER_CYCLE = 20
 
 # 测试API，建议抓哪个网站测哪个
-TEST_API = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=2145291155&containerid=1076032145291155&page=14'
+TEST_URL = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=2145291155&containerid=1076032145291155&page=14'
 
 # API配置
 API_HOST = '0.0.0.0'

proxypool/tester.py

@@ -28,20 +28,21 @@ class Tester(object):
         :param proxy: 
         :return: 
         """
-        async with aiohttp.ClientSession() as session:
+        conn = aiohttp.TCPConnector(verify_ssl=False)
+        async with aiohttp.ClientSession(connector=conn) as session:
             try:
                 if isinstance(proxy, bytes):
                     proxy = proxy.decode('utf-8')
                 real_proxy = 'http://' + proxy
                 print('正在测试', proxy)
-                async with session.get(TEST_API, proxy=real_proxy, timeout=15) as response:
+                async with session.get(TEST_URL, proxy=real_proxy, timeout=15) as response:
                     if response.status in VALID_STATUS_CODES:
                         self.redis.add(proxy)
                         print('代理可用', proxy)
                     else:
                         self.redis.decrease(proxy)
                         print('请求响应码不合法，IP', proxy)
-            except (ClientError, aiohttp.client_exceptions.ClientConnectorError, asyncio.TimeoutError):
+            except (ClientError, aiohttp.client_exceptions.ClientConnectorError, asyncio.TimeoutError, AttributeError):
                 if self.redis.exists(proxy):
                     self.redis.decrease(proxy)
                 print('代理请求失败', proxy)

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 
 setup(
-    name='proxy-pool',
+    name='proxypool',
     version='1.0.0',
     description='High performance proxy pool',
     long_description='A proxy pool project modified from WiseDoge/ProxyPool',
@@ -9,7 +9,7 @@ setup(
     author_email='[email protected]',
     url='https://github.com/Germey/ProxyPool',
     packages=[
-        'proxy-pool'
+        'proxypool'
     ],
     py_modules=['run'],
     include_package_data=True,
@@ -22,7 +22,7 @@ setup(
         'pyquery'
     ],
     entry_points={
-        'console_scripts': ['proxy_pool_run=run:cli']
+        'console_scripts': ['proxy_pool_run=run:main']
     },
     license='apache 2.0',
     zip_safe=False,