germey_ProxyPool/proxypool/getter.py

from proxypool.db import RedisClient
from proxypool.setting import PROXY_NUMBER_MAX
from proxypool.crawlers import __all__ as crawlers_cls


class Getter():
    """
    getter of proxypool
    """
    
    
    def __init__(self):
        """
        init db and crawlers
        """
        self.redis = RedisClient()
        self.crawlers_cls = crawlers_cls
        self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls]
    
    
    def is_full(self):
        """
        if proxypool if full
        return: bool
        """
        return self.redis.count() >= PROXY_NUMBER_MAX
    
    
    def run(self):
        """
        run crawlers to get proxy
        :return:
        """
        if self.is_full():
            return
        for crawler in self.crawlers:
            for proxy in crawler.crawl():
                print('proxy', proxy)
                self.redis.add(proxy)


if __name__ == '__main__':
    getter = Getter()
    getter.run()