refactor checkers, new fields(ip, city, country_code, region)

DevAlone · DevAlone · commit 9e6f7d5ad037 · 2018-02-05T18:35:14.000+03:00
diff --git a/checkers/base_checker.py b/checkers/base_checker.py
@@ -0,0 +1,98 @@
+import ssl
+
+import aiosocks
+import asyncio
+
+import async_requests
+
+from aiosocks.connector import ProxyConnector, ProxyClientRequest
+import aiohttp
+
+
+class CheckerResult:
+    # TODO: change to properties with validation
+    ipv4 = None
+    ipv6 = None
+    city = None
+    region = None
+    country_code = None
+    # tuple
+    location_coordinates = None
+    organization_name = None
+
+    def update_from_other(self, other):
+        def set_attr_if_is_not_none(attribute_name, first_obj, second_obj):
+            second_val = getattr(second_obj, attribute_name)
+            if second_val is not None:
+                setattr(first_obj, attribute_name, second_val )
+
+        set_attr_if_is_not_none('ipv4', self, other)
+        set_attr_if_is_not_none('ipv6', self, other)
+        set_attr_if_is_not_none('city', self, other)
+        set_attr_if_is_not_none('region', self, other)
+        set_attr_if_is_not_none('country_code', self, other)
+        set_attr_if_is_not_none('location_coordinates', self, other)
+        set_attr_if_is_not_none('organization_name', self, other)
+
+
+class BaseChecker:
+    def __init__(self, url=None, request_type="GET", timeout=10):
+        self.request_type = request_type
+        self.timeout = timeout
+        self.url = url
+
+    async def check(self, proxy_address: str, timeout: int=None) -> tuple:
+        """
+        Checks proxy and returns additional information if such was provided by checker server
+
+        :param proxy_address: string representing proxy ("http://user@qwerty@127.0.0.1:8080")
+        :param timeout: overrides timeout if not None
+        :return: tuple where first item is bool indication whether proxy is working or not
+        and second one is additional information structure with information like white ip address, country and so on
+        """
+
+        checker_result = CheckerResult()
+        if self.url is None:
+            raise Exception()
+
+        headers = {
+            'User-Agent': async_requests.get_random_user_agent()
+        }
+
+        conn = ProxyConnector(remote_resolve=True)
+
+        timeout = timeout if timeout is not None else self.timeout
+
+        with aiohttp.ClientSession(connector=conn, request_class=ProxyClientRequest) as session:
+            try:
+                async with session.request(
+                            self.request_type, self.url, proxy=proxy_address, timeout=timeout, headers=headers) as \
+                        response:
+                    is_working = await self._check(response, checker_result)
+            except (aiohttp.client_exceptions.ServerDisconnectedError,
+                    aiohttp.client_exceptions.ClientHttpProxyError,
+                    aiohttp.client_exceptions.ClientProxyConnectionError,
+                    aiohttp.client_exceptions.ClientResponseError,
+                    aiosocks.errors.SocksError,
+                    aiosocks.SocksError,
+                    asyncio.TimeoutError,
+                    aiohttp.client_exceptions.ClientOSError,
+                    ssl.CertificateError,
+                    ) as ex:
+                is_working = False
+                message = str(ex)
+                if "file" in message:
+                    print(message)
+                # TODO: check for "Too many open files"
+
+        return is_working, checker_result
+
+    async def _check(self, response: aiohttp.ClientResponse, checker_result: CheckerResult) -> bool:
+        """
+        Implement this method. It will get response from url with http method you provided in constructor
+
+        :param response: aiohttp response
+        :param checker_result: fill this structure with information like ip address
+        :return: whether proxy is working or not
+        """
+        raise NotImplemented()
diff --git a/checkers/d3d_info_checker.py b/checkers/d3d_info_checker.py
@@ -0,0 +1,11 @@
+import aiohttp
+
+from checkers.base_checker import BaseChecker, CheckerResult
+
+
+class D3DInfoChecker(BaseChecker):
+    def __init__(self, timeout=10):
+        super(D3DInfoChecker, self).__init__("https://pikagraphs.d3d.info/OK/", timeout=timeout)
+
+    async def _check(self, response: aiohttp.ClientResponse, checker_result: CheckerResult):
+        return (await response.text()) == 'OK'
diff --git a/checkers/ipinfo_io_checker.py b/checkers/ipinfo_io_checker.py
@@ -0,0 +1,30 @@
+import traceback
+
+import aiohttp
+
+from checkers.base_checker import BaseChecker, CheckerResult
+
+
+class IPInfoIOChecker(BaseChecker):
+    def __init__(self, timeout=10):
+        super(IPInfoIOChecker, self).__init__("https://ipinfo.io/json", timeout=timeout)
+
+    async def _check(self, response: aiohttp.ClientResponse, checker_result: CheckerResult):
+        if response.status != 200:
+            return False
+
+        try:
+            json_result = await response.json()
+            checker_result.ipv4 = json_result['ip']
+            checker_result.city = json_result['city']
+            checker_result.region = json_result['region']
+            checker_result.country_code = json_result['country']
+            checker_result.location_coordinates = tuple(float(x) for x in json_result['loc'].split(','))
+            checker_result.organization_name = json_result['org']
+            return True
+        # TODO: change to actual exception
+        except BaseException as ex:
+            traceback.print_exc()
+            raise ex
+
+        return False
diff --git a/models.py b/models.py
@@ -35,6 +35,11 @@ class Proxy(Base):
     bad_uptime = Column(Integer, nullable=True, default=None)
     # in microseconds
     response_time = Column(Integer, nullable=True, default=None)
+    _white_ipv4 = Column(String(16), nullable=True)
+    _white_ipv6 = Column(String(16), nullable=True)
+    city = Column(String(), nullable=True)
+    region = Column(String(), nullable=True)
+    country_code = Column(String(3), nullable=True)
 
     def get_raw_protocol(self):
         return self.raw_protocol
@@ -55,6 +60,22 @@ def protocol(self, protocol):
     def bad_proxy(self):
         return self.number_of_bad_checks > 0
 
+    @property
+    def white_ipv4(self):
+        return self._white_ipv4
+
+    @white_ipv4.setter
+    def white_ipv4(self, value):
+        self._white_ipv4 = value
+
+    @property
+    def white_ipv6(self):
+        return self._white_ipv6
+
+    @white_ipv6.setter
+    def white_ipv6(self, value):
+        self._white_ipv6 = value
+
     def to_url(self, protocol=None):
         address = protocol if protocol is not None else self.PROTOCOLS[int(self.raw_protocol)]
         address += "://"
diff --git a/processor.py b/processor.py
@@ -2,6 +2,7 @@
 
 import collectors_list
 import proxy_utils
+from checkers.base_checker import CheckerResult
 
 from proxy_py import settings
 from models import Proxy
@@ -215,7 +216,9 @@ async def process_proxy(self, raw_protocol: int, auth_data: str, domain: str, po
 
         try:
             start_checking_time = time.time()
-            check_result = await proxy_utils.check_proxy(proxy_url, aiohttp_proxy_check_session)
+            check_result, checker_additional_info = await proxy_utils.check_proxy(
+                proxy_url, aiohttp_proxy_check_session)
+
             end_checking_time = time.time()
 
             if check_result:
@@ -226,7 +229,8 @@ async def process_proxy(self, raw_protocol: int, auth_data: str, domain: str, po
                     domain,
                     port,
                     start_checking_time,
-                    end_checking_time
+                    end_checking_time,
+                    checker_additional_info
                 )
             else:
                 self.logger.debug("proxy {0} doesn't work".format(proxy_url))
@@ -260,7 +264,7 @@ async def process_proxy(self, raw_protocol: int, auth_data: str, domain: str, po
 
     @staticmethod
     def create_or_update_proxy(raw_protocol: Proxy.PROTOCOLS, auth_data, domain, port,
-                               start_checking_time, end_checking_time):
+                               start_checking_time, end_checking_time, additional_info: CheckerResult):
         if raw_protocol is None or domain is None or port is None or auth_data is None or start_checking_time is None\
                 or end_checking_time is None:
             raise Exception("Bad arguments")
@@ -297,6 +301,18 @@ def create_or_update_proxy(raw_protocol: Proxy.PROTOCOLS, auth_data, domain, por
         proxy.number_of_bad_checks = 0
         proxy.last_check_time = int(time.time())
 
+        if additional_info.ipv4 is not None:
+            proxy.white_ipv4 = additional_info.ipv4
+
+        if additional_info.city is not None:
+            proxy.city = additional_info.city
+
+        if additional_info.region is not None:
+            proxy.region = additional_info.region
+
+        if additional_info.country_code is not None:
+            proxy.country_code = additional_info.country_code.strip().lower()
+
         checking_time = int(end_checking_time - start_checking_time)
         if checking_time > settings.PROXY_CHECKING_TIMEOUT:
             checking_time = settings.PROXY_CHECKING_TIMEOUT
diff --git a/proxy_py/_settings.py b/proxy_py/_settings.py
@@ -1,3 +1,6 @@
+from checkers.d3d_info_checker import D3DInfoChecker
+from checkers.ipinfo_io_checker import IPInfoIOChecker
+
 import os
 
 DATABASE_CONNECTION_ARGS = (
@@ -20,7 +23,7 @@
 BAD_PROXY_CHECKING_PERIOD = 2 * 60 * 60
 DEAD_PROXY_CHECKING_PERIOD = 24 * 60 * 60
 DEAD_PROXY_THRESHOLD = 6
-REMOVE_ON_N_BAD_CHECKS = DEAD_PROXY_THRESHOLD + 21
+REMOVE_ON_N_BAD_CHECKS = DEAD_PROXY_THRESHOLD + 64
 PROXY_CHECKING_TIMEOUT = 20
 
 PROXY_PROVIDER_SERVER_ADDRESS = {
@@ -29,11 +32,11 @@
 }
 
 _PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG = {
-    'fields': ['address', 'protocol', 'auth_data', 'domain', 'port', 'last_check_time',
-               'number_of_bad_checks', 'bad_proxy', 'uptime', 'response_time'],
+    'fields': ['address', 'protocol', 'auth_data', 'domain', 'port', 'last_check_time', 'number_of_bad_checks',
+               'bad_proxy', 'uptime', 'response_time', 'white_ipv4', 'white_ipv6', 'city', 'country_code', 'region'],
     'filter_fields': ['last_check_time', 'protocol', 'number_of_bad_checks', 'bad_proxy', 'uptime',
                       'response_time'],
-    'order_by_fields': ['last_check_time', 'number_of_bad_checks', 'uptime', 'response_time'],
+    'order_by_fields': ['last_check_time', 'number_of_bad_checks', 'uptime', 'response_time', 'country_code'],
     'default_order_by_fields': ['response_time', ],
 }
 
@@ -46,3 +49,12 @@
         }
     }
 }
+
+# can be either "or" or "and". "Or" means to consider proxy as working if at least one checker checked it OK,
+# if "and" proxy is considered as working only when all checkers said that proxy is good
+PROXY_CHECKING_CONDITION = 'or'
+
+PROXY_CHECKERS = [
+    IPInfoIOChecker(PROXY_CHECKING_TIMEOUT),
+    D3DInfoChecker(PROXY_CHECKING_TIMEOUT),
+]
diff --git a/proxy_utils.py b/proxy_utils.py
@@ -1,62 +1,26 @@
-import traceback
+from checkers.base_checker import CheckerResult
+from proxy_py import settings
 
-import sys
 
-from proxy_py import settings
-import async_requests
-
-import asyncio
-import ssl
-import aiohttp
-import aiosocks
-import os
-
-from aiosocks.connector import ProxyConnector, ProxyClientRequest
-
-
-# TODO: add multiple checks with several sites
-async def check_proxy(proxy_url: str, session=None, timeout=None):
-    try:
-        res = await _request(
-            'get',
-            'https://pikagraphs.d3d.info/OK/',
-            # 'https://wtfismyip.com/text',
-            proxy_url,
-            settings.PROXY_CHECKING_TIMEOUT if timeout is None else timeout,
-            session
-        )
-
-        if res.status == 200 and res.text == "OK":
-            return True
-    except (aiohttp.client_exceptions.ServerDisconnectedError,
-            aiohttp.client_exceptions.ClientHttpProxyError,
-            aiohttp.client_exceptions.ClientProxyConnectionError,
-            aiohttp.client_exceptions.ClientResponseError,
-            aiosocks.errors.SocksError,
-            aiosocks.SocksError,
-            asyncio.TimeoutError,
-            aiohttp.client_exceptions.ClientOSError,
-            ssl.CertificateError,
-            ) as ex:
-        message = str(ex)
-        if "file" in message:
-            print(message)
-        # TODO: check for "Too many open files"
-        return False
-
-    return False
-
-
-async def _request(method, url, proxy_url, timeout, session: aiohttp.ClientSession=None):
-    headers = {
-        'User-Agent': async_requests.get_random_user_agent()
-    }
-
-    conn = ProxyConnector(remote_resolve=True)
-
-    with aiohttp.ClientSession(connector=conn, request_class=ProxyClientRequest) as session:
-        async with session.request(method, url, proxy=proxy_url, timeout=timeout) as response:
-            # async with session.request(method, url, timeout=timeout, headers=headers, proxy=proxy_url) as response:
-            status = response.status
-            text = await response.text()
-            return async_requests.Response(status, text)
+async def check_proxy(proxy_url: str, timeout=None) -> tuple:
+    results = []
+
+    for checker in settings.PROXY_CHECKERS:
+        result = await checker.check(proxy_url, timeout=timeout)
+        if result[0] and settings.PROXY_CHECKING_CONDITION == 'or':
+            return result
+
+        results.append(result)
+
+    additional_information = CheckerResult()
+
+    if settings.PROXY_CHECKING_CONDITION == 'and':
+        for result in results:
+            if not result[0]:
+                return False, additional_information
+
+            additional_information.update_from_other(result[1])
+
+        return True, additional_information
+
+    return False, additional_information
