better proxy validation, parsers, some of guide

DevAlone · DevAlone · commit 7f775ae5f8ff · 2018-10-01T00:08:23.000+03:00
diff --git a/collectors/web/cn/89ip/collector.py b/collectors/web/cn/89ip/collector.py
@@ -1,4 +1,6 @@
 from collectors.abstract_collector import AbstractCollector
+from parsers.regex_parser import RegexParser
+
 import http_client
 
 
@@ -18,6 +20,9 @@ def __init__(self):
 
     async def collect(self):
         url = 'http://www.89ip.cn/tqdl.html?num=9999&address=&kill_address=&port=&kill_port=&isp='
-        html = await http_client.get_text(url)
-
         return []
+        # html = await http_client.get_text(url)
+        with open('/tmp/mock') as f:
+            html = f.read()
+
+        return RegexParser().parse(html)
diff --git a/collectors_list.py b/collectors_list.py
@@ -60,19 +60,6 @@ async def init():
             )
 
 
-# def get_collector_state(module_name: str):
-#     try:
-#         collector_state = asyncio.get_event_loop().run_until_complete(db.get(
-#             CollectorState.select().where(
-#                 CollectorState.identifier == module_name
-#             )
-#         ))
-#     except CollectorState.DoesNotExist:
-#         raise CollectorNotFoundException()
-#
-#     return collector_state
-
-
 def get_collector_of_module_name(module_name: str):
     if module_name not in collectors:
         raise CollectorNotFoundException(
diff --git a/docs/source/guides/how_to_create_collector.rst b/docs/source/guides/how_to_create_collector.rst
@@ -4,14 +4,14 @@ proxy_py How to create collector
 Collector is a class which is used to parse proxies from web page or another source.
 All collectors are inherited from `collectors.abstract_collector.AbstractCollector`,
 also there is `collectors.pages_collector.PagesCollector` which is used for paginated sources.
-It's better to understand through examples.
+It's always better to learn through the examples.
 
 Simple collector
 ****************
 
 Let's start with the simplest collector we can imagine,
-it will be from the page http://www.89ip.cn/ti.html as you can see,
-it sends form as GET request to this url
+it will be collecting from the page http://www.89ip.cn/ti.html
+as you can see, it sends form as GET request to this url
 http://www.89ip.cn/tqdl.html?num=9999&address=&kill_address=&port=&kill_port=&isp=
 
 Firstly we can try to check that these proxies are really good.
@@ -25,20 +25,23 @@ You're gonna get something like this:
 
 `++++++++++++++++++++++-+++++-+++++++++++++++++++++++++++-++++++-++++-+++++++++++++++++++++++++++++++--+++++++-+++++++-++-+-+++-+++++++++-+++++++++++++++++++++--++--+-++++++++++++++++-+++--+++-+-+++++++++++++++++--++++++++++++-+++++-+++-++++++++-+++++-+-+++++++-++-+--++++-+++-++++++++++-++++--+++++++-+++++++-++--+++++-+-+++++++++++++++++++++-++-+++-+++--++++--+++-+++++++-+++++++-+++++++++++++++---+++++-+++++++++-+++++-+-++++++++++++-+--+++--+-+-+-++-+++++-+++--++++++-+++++++++++--+-+++-+-++++--+++++--+++++++++-+-+-++++-+-++++++++++++++-++-++++++--+--++++-+-++--++--+++++-++-+++-++++--++--+---------+--+--++--------+++-++-+--++++++++++++++++-+++++++++-+++++++--+--+--+-+-+++---++------------------+--+----------+-+-+--++-+----------+-------+--+------+----+-+--+--++----+--+-++++++-++-+++`
 
-+ means working proxy with at least one protocol,
-- means not working, the result above is perfect, so many good proxies.
+"\+" means working proxy with at least one protocol, "\-" means not working, the result above is perfect, so many good proxies.
 
-Note: working means proxy respond with timeout set in settings, if you increase it you'll get more proxies.
+Note: working means proxy respond with timeout set in settings,
+if you increase it, you're likely to get more proxies.
 
-Alright, let's code it!
+Alright, let's code!
 
-We need to place our collector inside `collectors/web/` directory using reversed domain path,
+We need to place our collector inside `collectors/web/`
+directory using reversed domain path,
 it will be `collectors/web/cn/89ip/collector.py`
 
-To make class be collector we need to declare variable `__collector__`
+To make class be a collector we need to declare a variable
+`__collector__`
 
 Note: name of file and name of class don't make sense,
-you can declare as many files and classes in each file per domain as you want
+you can declare as many files and classes in each file
+per domain as you want
 
 .. code-block:: python
 
@@ -48,22 +51,26 @@ you can declare as many files and classes in each file per domain as you want
     class Collector(AbstractCollector):
         __collector__ = True
 
-We can override default processing period in constructor like this:
+We can override default processing period in constructor
+like this:
 
 .. code-block:: python
 
     def __init__(self):
         super(Collector, self).__init__()
-        self.processing_period = 30 * 60  # 30 minutes
+        # 30 minutes
+        self.processing_period = 30 * 60
         '''
         floating period means proxy_py will be changing
         period to not make extra requests and handle
-        new proxies in time, you don't need to change
+        new proxies in time, you don't need to disable
         it in most cases
         '''
         # self.floating_processing_period = False
 
 
+The last step is to implement `collect()` method.
+
 Paginated collector
 *******************
 
diff --git a/main.py b/main.py
@@ -8,9 +8,40 @@
 
 import asyncio
 import logging
+import argparse
 
+from tools import test_collector
+
+test_collector_path = None
+main_logger = None
+
+
+def process_cmd_arguments():
+    global test_collector_path
+
+    def str_to_bool(value):
+        if value.lower() in ("yes", "true", "t", "y", "1"):
+            return True
+        elif value.lower() in ("no", "false", "f", "n", "0"):
+            return False
+        else:
+            raise argparse.ArgumentTypeError("Boolean value expected.")
+
+    cmd_parser = argparse.ArgumentParser()
+    cmd_parser.add_argument("--debug", type=str_to_bool, help="override settings' debug value")
+    cmd_parser.add_argument("--test-collector", help="test collector with a given path")
+
+    args = cmd_parser.parse_args()
+
+    if args.debug is not None:
+        settings.DEBUG = args.debug
+
+    test_collector_path = args.test_collector
+
+
+def prepare_loggers():
+    global main_logger
 
-if __name__ == "__main__":
     asyncio_logger = logging.getLogger("asyncio")
     asyncio_logger_file_handler = logging.FileHandler("logs/asyncio.log")
     asyncio_logger_file_handler.setLevel(logging.DEBUG)
@@ -43,8 +74,16 @@
 
     main_logger.addHandler(logger_file_handler)
 
+
+def main():
+    process_cmd_arguments()
+    prepare_loggers()
+
     loop = asyncio.get_event_loop()
 
+    if test_collector_path is not None:
+        return loop.run_until_complete(test_collector.run(test_collector_path))
+
     proxy_processor = Processor.get_instance()
 
     proxy_provider_server = ProxyProviderServer(
@@ -61,6 +100,15 @@
             statistics.worker(),
         ]))
         BaseChecker.clean()
+    except KeyboardInterrupt:
+        pass
     except BaseException as ex:
         main_logger.exception(ex)
         print("critical error happened, see logs/main.log")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/parsers/regex_parser.py b/parsers/regex_parser.py
@@ -0,0 +1,47 @@
+import re
+
+
+# TODO: add ipv6 addresses, make domain checking better
+_0_TO_255_REGEX = r"([0-9]|[1-8][0-9]|9[0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])"
+DOMAIN_LETTER_REGEX = r"[a-zA-Z0-9_\-]"
+PROXY_FIND_REGEX = \
+    r"((?P<protocol>(http|socks4|socks5))://)?" \
+    r"((?P<auth_data>[a-zA-Z0-9_\.]+:[a-zA-Z0-9_\.]+)@)?" \
+    r"(?P<domain>" + \
+    r"(" + _0_TO_255_REGEX + "\.){3}" + _0_TO_255_REGEX + \
+    r"|" + DOMAIN_LETTER_REGEX + r"+(\.[a-zA-Z]" + DOMAIN_LETTER_REGEX + r"+)+):" \
+    r"(?P<port>(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|999[0-9]|99[0-8][0-9]|9[0-8][0-9]{2}|[1-8][0-9]{3}|99[0-9]|9[0-8][0-9]|[1-8][0-9]{2}|9[0-9]|[1-8][0-9]|[1-9]))"
+PROXY_VALIDATE_REGEX = '^' + PROXY_FIND_REGEX + '/?$'
+
+
+class RegexParser:
+    '''
+    It's used to scratch proxies from text by regular expression,
+    you can pass your own expression(see docstring for parse method)
+    '''
+
+    def __init__(self, expression=PROXY_FIND_REGEX, flags=0):
+        '''
+
+        :param expression: expression which is used to parse proxies with named groups:
+        (?P<protocol>) - for proxy protocol (socks4/5, http)
+        (?P<auth_data>) - authentication data (login and password)
+        (?P<domain>) - IP or domain
+        (?P<port>) - port
+        There is a default value which parses proxies like these ones:
+        127.0.0.1:9060
+        test.proxy.com:8080
+        socks4://8.8.8.8:65000
+        http://user:password@proxy.test.info/
+
+        :param flags: flags that are passed to re.compile
+        '''
+
+        flags |= re.MULTILINE
+        self.expression = expression
+        self.regex = re.compile(expression, flags)
+
+    def parse(self, text: str) -> list:
+        for match in self.regex.finditer(text):
+            match = match.groupdict()
+            yield match["domain"] + ':' + match["port"]
diff --git a/processor.py b/processor.py
@@ -1,4 +1,6 @@
+import proxy_validator
 from checkers.base_checker import CheckerResult
+from parsers.regex_parser import PROXY_VALIDATE_REGEX
 from proxy_py import settings
 from models import Proxy, CollectorState, db
 
@@ -11,16 +13,6 @@
 import peewee
 
 
-# TODO: add ipv6 addresses, make domain checking better
-_0_TO_255_REGEX = r"([0-9]|[1-8][0-9]|9[0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])"
-DOMAIN_LETTER_REGEX = r"[a-zA-Z0-9_\-]"
-PROXY_VALIDATE_REGEX = \
-    r"^((?P<protocol>(http|socks4|socks5))://)?" \
-    r"((?P<auth_data>[a-zA-Z0-9_\.]+:[a-zA-Z0-9_\.]+)@)?" \
-    r"(?P<domain>" + \
-        r"(" + _0_TO_255_REGEX + "\.){3}" + _0_TO_255_REGEX + \
-        r"|" + DOMAIN_LETTER_REGEX + r"+(\.[a-zA-Z]" + DOMAIN_LETTER_REGEX + r"+)*):" \
-    r"(?P<port>([1-9]|[1-8][0-9]|9[0-9]|[1-8][0-9]{2}|9[0-8][0-9]|99[0-9]|[1-8][0-9]{3}|9[0-8][0-9]{2}|99[0-8][0-9]|999[0-9]|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5]))/?$"
 
 LOGGERS_FORMAT = "%(levelname)s ~ %(asctime)s ~ %(funcName)30s() ~ %(message)s"
 
@@ -213,59 +205,53 @@ async def process_raw_proxies(self, proxies, collector_id):
     async def process_raw_proxy(self, proxy, collector_id):
         self.logger.debug("processing raw proxy \"{}\"".format(proxy))
 
-        matches = re.match(PROXY_VALIDATE_REGEX, proxy)
-        if matches:
-            matches = matches.groupdict()
-            auth_data = matches["auth_data"]
-            domain = matches["domain"]
-            port = matches["port"]
+        try:
+            _, auth_data, domain, port = proxy_validator.retrieve(proxy)
+        except proxy_validator.ValidationError as ex:
+            self.collectors_logger.error(
+                "Collector with id \"{}\" returned bad raw proxy \"{}\". "
+                "Message: {}".format(collector_id, proxy, ex)
+            )
+            return
 
-            if auth_data is None:
-                auth_data = ""
 
-            if domain is None or port is None:
-                self.collectors_logger.error(
-                    "Bad raw proxy \"{}\" from collector \"{}\"".format(proxy, collector_id)
+        # don't care about protocol
+        try:
+            proxy = await db.get(
+                Proxy.select().where(
+                    Proxy.auth_data == auth_data,
+                    Proxy.domain == domain,
+                    Proxy.port == port,
                 )
-                return
+            )
 
-            # don't care about protocol
-            try:
-                proxy = await db.get(
-                    Proxy.select().where(
-                        Proxy.auth_data == auth_data,
-                        Proxy.domain == domain,
-                        Proxy.port == port,
-                    )
+            if proxy.last_check_time + settings.PROXY_NOT_CHECKING_PERIOD >= time.time():
+                proxy_short_address = ""
+                if auth_data:
+                    proxy_short_address += auth_data + "@"
+
+                proxy_short_address += "{}:{}".format(domain, port)
+
+                self.logger.debug(
+                    "skipping proxy \"{}\" from collector \"{}\"".format(
+                        proxy_short_address, collector_id)
                 )
+                return
+        except Proxy.DoesNotExist:
+            pass
 
-                if proxy.last_check_time + settings.PROXY_NOT_CHECKING_PERIOD >= time.time():
-                    proxy_short_address = ""
-                    if auth_data:
-                        proxy_short_address += auth_data + "@"
+        for raw_protocol in range(len(Proxy.PROTOCOLS)):
+            while not self.good_proxies_are_processed:
+                # TODO: find a better way
+                await asyncio.sleep(0.1)
 
-                    proxy_short_address += "{}:{}".format(domain, port)
+            new_proxy = Proxy()
+            new_proxy.raw_protocol = raw_protocol
+            new_proxy.auth_data = auth_data
+            new_proxy.domain = domain
+            new_proxy.port = port
 
-                    self.logger.debug(
-                        "skipping proxy \"{}\" from collector \"{}\"".format(
-                            proxy_short_address, collector_id)
-                    )
-                    return
-            except Proxy.DoesNotExist:
-                pass
-
-            for raw_protocol in range(len(Proxy.PROTOCOLS)):
-                while not self.good_proxies_are_processed:
-                    # TODO: find a better way
-                    await asyncio.sleep(0.1)
-
-                new_proxy = Proxy()
-                new_proxy.raw_protocol = raw_protocol
-                new_proxy.auth_data = auth_data
-                new_proxy.domain = domain
-                new_proxy.port = port
-
-                await self.add_proxy_to_queue(new_proxy, collector_id)
+            await self.add_proxy_to_queue(new_proxy, collector_id)
 
     async def process_proxy(self, raw_protocol: int, auth_data: str, domain: str, port: int, collector_id):
         async with self.proxies_semaphore:
diff --git a/proxy_validator.py b/proxy_validator.py
diff --git a/tests/test_proxy_validation_regex.py b/tests/test_proxy_validation_regex.py
diff --git a/tools/test_collector.py b/tools/test_collector.py
