update docker compose

.dockerignore

@@ -0,0 +1,131 @@
+# Created by .ignore support plugin (hsz.mobi)
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

Dockerfile

@@ -0,0 +1,5 @@
+FROM python:3.6
+WORKDIR /app
+COPY . .
+RUN pip install -r requirements.txt
+CMD ["supervisord", "-c", "supervisord.conf"]

README.md

@@ -1,14 +1,20 @@
 # ProxyPool
 
+![](https://img.shields.io/badge/python-3.6%2B-brightgreen)
+
+Welcome to ProxyPool.
+
 ## Requirements
 
-* Docker 
+Below is a list of requirements for ProxyPool.
+
+* Docker
+* Docker-Compose
 
   or 
 
 * Python: >=3.6
 * Redis
-* Environment: Virtual Env
 
 ## Run with Docker
 
@@ -70,7 +76,9 @@ python3 run.py --processor tester
 python3 run.py --processor server
 ```
 
-### Usage
+## Usage
 
 After running the ProxyPool, you can visit 
-[http://localhost:5555/random](http://localhost:5555/) to access random proxy.  
+[http://localhost:5555/random](http://localhost:5555/) to access random proxy.  
+
+Just enjoy it.

docker-compose.yml

@@ -0,0 +1,18 @@
+version: '3'
+services:
+  redis:
+    image: redis:alpine
+    container_name: redis
+    command: redis-server
+    ports:
+      - "6379:6379"
+    restart: always
+  proxypool:
+    build: .
+    image: 'germey/proxypool'
+    container_name: proxypool
+    ports:
+      - "5555:5555"
+    restart: always
+    environment:
+      REDIS_HOST: redis

proxypool/scheduler.py

@@ -23,6 +23,9 @@ class Scheduler():
         """
         run tester
         """
+        if not ENABLE_TESTER:
+            logger.info('tester not enabled, exit')
+            return
         tester = Tester()
         loop = 0
         while True:
@@ -35,6 +38,9 @@ class Scheduler():
         """
         run getter
         """
+        if not ENABLE_GETTER:
+            logger.info('getter not enabled, exit')
+            return
         getter = Getter()
         loop = 0
         while True:
@@ -47,6 +53,9 @@ class Scheduler():
         """
         run server for api
         """
+        if not ENABLE_SERVER:
+            logger.info('server not enabled, exit')
+            return
         app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
     
     def run(self):

proxypool/setting.py

@@ -3,6 +3,8 @@ from os.path import dirname, abspath, join
 from environs import Env
 from loguru import logger
 
+from proxypool.utils.parse import parse_redis_connection_string
+
 
 env = Env()
 env.read_env()
@@ -29,10 +31,10 @@ REDIS_PORT = env.int('REDIS_PORT', 6379)
 # redis password, if no password, set it to None
 REDIS_PASSWORD = env.str('REDIS_PASSWORD', None)
 # redis connection string, like redis://[password]@host:port or rediss://[password]@host:port
-# REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None)
+REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None)
 
-# if REDIS_CONNECTION_STRING:
-#     REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING)
+if REDIS_CONNECTION_STRING:
+    REDIS_HOST, REDIS_PORT, REDIS_PASSWORD = parse_redis_connection_string(REDIS_CONNECTION_STRING)
 
 # redis hash table key name
 REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal')

requirements.txt

@@ -7,3 +7,5 @@ requests==2.22.0
 loguru==0.3.2
 pyquery==1.4.0
 attr==0.3.1
+supervisor==4.1.0
+redis==2.10.6

run.py

@@ -5,7 +5,6 @@ import argparse
 parser = argparse.ArgumentParser(description='ProxyPool')
 parser.add_argument('--processor', type=str, help='processor to run')
 args = parser.parse_args()
-print('args', args)
 
 if __name__ == '__main__':
     # if processor set, just run it

supervisord.conf

@@ -0,0 +1,17 @@
+[supervisord]
+nodaemon=true
+
+[program:tester]
+process_name=tester
+command=python3 run.py --processor tester
+directory=/app
+
+[program:getter]
+process_name=getter
+command=python3 run.py --processor getter
+directory=/app
+
+[program:server]
+process_name=server
+command=python3 run.py --processor server
+directory=/app