big refactoring of server, api versioning

Author: DevAlone

main.py

@@ -43,16 +43,18 @@ async def proxy_counter():
 
 
 if __name__ == "__main__":
+    loop = asyncio.get_event_loop()
+
     proxy_processor = Processor()
 
-    proxy_provider_server = ProxyProviderServer.get_proxy_provider_server(
+    proxy_provider_server = ProxyProviderServer(
         settings.PROXY_PROVIDER_SERVER_ADDRESS['HOST'],
         settings.PROXY_PROVIDER_SERVER_ADDRESS['PORT'],
-        None,  # proxy_processor,
+        proxy_processor,
     )
 
-    loop = asyncio.get_event_loop()
     loop.run_until_complete(proxy_provider_server.start(loop))
+
     loop.run_until_complete(asyncio.wait([
         proxy_processor.exec(),
         proxy_counter(),

proxy_py/_settings.py

@@ -70,3 +70,5 @@
     IPInfoIOChecker(PROXY_CHECKING_TIMEOUT),
     D3DInfoChecker(PROXY_CHECKING_TIMEOUT),
 ]
+
+TEMPLATES_PATH = "server/templates"

server/api_request_handler.py renamed to server/api_v1/api_request_handler.py

@@ -1,5 +1,5 @@
-from server.requests_to_models.request_parser import RequestParser, ParseError
-from server.requests_to_models.request_executor import RequestExecutor, ExecutionError
+from .requests_to_models.request_parser import RequestParser, ParseError
+from .requests_to_models.request_executor import RequestExecutor, ExecutionError
 from proxy_py import settings
 
 

server/api_v1/app.py

@@ -0,0 +1,53 @@
+from server.base_app import BaseApp
+from .api_request_handler import ApiRequestHandler
+from aiohttp import web
+
+import json
+import aiohttp
+
+
+class App(BaseApp):
+    def __init__(self, *args, **kwargs):
+        super(App, self).__init__(*args, **kwargs)
+
+        self.request_handler = ApiRequestHandler(self.logger)
+
+    async def setup_router(self):
+        self.app.router.add_post('/', self.post)
+
+    async def post(self, request):
+        client_address = request.transport.get_extra_info('peername')
+        host, port = (None, None)
+
+        if client_address is not None:
+            host, port = client_address
+        else:
+            client_address = (host, port)
+
+        data = await request.read()
+
+        with open("logs/apiv1_server_connections", 'a') as f:
+            f.write("client - {}:{}, data - {}\n".format(host, port, data))
+
+        try:
+            data = json.loads(data.decode())
+
+            response = await self.request_handler.handle(client_address, data)
+        except ValueError:
+            response = {
+                'status': 'error',
+                'status_code': 400,
+                'error_message': "Your request doesn't look like request",
+            }
+
+        if 'status_code' in response:
+            status_code = response['status_code']
+        else:
+            if response['status'] != 'ok':
+                status_code = 500
+            else:
+                status_code = 200
+
+            response['status_code'] = status_code
+
+        return aiohttp.web.json_response(response, status=status_code)

server/requests_to_models/request.py renamed to server/api_v1/requests_to_models/request.py

@@ -1,6 +1,6 @@
 from proxy_py import settings
 from models import db
-from server.requests_to_models.request import Request, GetRequest, CountRequest, FetchRequest
+from ..requests_to_models.request import Request, GetRequest, CountRequest, FetchRequest
 import importlib
 
 

server/requests_to_models/request_executor.py renamed to server/api_v1/requests_to_models/request_executor.py

@@ -1,4 +1,4 @@
-from server.requests_to_models.request import Request, GetRequest, CountRequest
+from ..requests_to_models.request import Request, GetRequest, CountRequest
 
 import string
 import copy

server/requests_to_models/request_parser.py renamed to server/api_v1/requests_to_models/request_parser.py

@@ -0,0 +1,36 @@
+import asyncio
+
+from proxy_py import settings
+from aiohttp import web
+
+import abc
+import json
+import aiohttp
+import aiohttp_jinja2
+import jinja2
+
+
+class BaseApp:
+    def __init__(self, logger=None):
+        self.logger = logger
+        self._app = web.Application()
+
+        aiohttp_jinja2.setup(self.app, loader=jinja2.FileSystemLoader(
+            settings.TEMPLATES_PATH
+        ))
+
+    async def init(self):
+        """Call it before anything else"""
+        await self.setup_router()
+        await self.setup_middlewares()
+
+    @abc.abstractmethod
+    async def setup_router(self):
+        pass
+
+    async def setup_middlewares(self):
+        pass
+
+    @property
+    def app(self):
+        return self._app

server/base_app.py

@@ -0,0 +1,102 @@
+from proxy_py import settings
+from models import db, Proxy, ProxyCountItem, CollectorState
+from server.base_app import BaseApp
+from aiohttp import web
+
+import time
+import datetime
+import functools
+import aiohttp_jinja2
+
+
+def get_response_wrapper(template_name):
+    def decorator_wrapper(func):
+        @functools.wraps(func)
+        @aiohttp_jinja2.template(template_name)
+        async def wrap(self, *args, **kwargs):
+            good_proxies_count = await db.count(
+                Proxy.select().where(Proxy.number_of_bad_checks == 0)
+            )
+
+            bad_proxies_count = await db.count(
+                Proxy.select().where(
+                    Proxy.number_of_bad_checks > 0,
+                    Proxy.number_of_bad_checks < settings.DEAD_PROXY_THRESHOLD,
+                )
+            )
+
+            dead_proxies_count = await db.count(
+                Proxy.select().where(
+                    Proxy.number_of_bad_checks >= settings.DEAD_PROXY_THRESHOLD,
+                )
+            )
+
+            response = {
+                "bad_proxies_count": bad_proxies_count,
+                "good_proxies_count": good_proxies_count,
+                "dead_proxies_count": dead_proxies_count,
+            }
+
+            response.update(await func(self, *args, **kwargs))
+
+            return response
+        return wrap
+
+    return decorator_wrapper
+
+
+class App(BaseApp):
+    async def setup_router(self):
+        self.app.router.add_get('/get/proxy/', self.get_proxies_html)
+        self.app.router.add_get('/get/proxy_count_item/', self.get_proxy_count_items_html)
+        self.app.router.add_get('/get/collector_state/', self.get_collector_state_html)
+        self.app.router.add_get('/get/best/http/proxy/', self.get_best_http_proxy)
+
+    @get_response_wrapper("collector_state.html")
+    async def get_collector_state_html(self, request):
+        return {
+            "collector_states": list(await db.execute(CollectorState.select())),
+        }
+
+    @get_response_wrapper("proxies.html")
+    async def get_proxies_html(self, request):
+        proxies = await db.execute(
+            Proxy.select().where(Proxy.number_of_bad_checks == 0).order_by(Proxy.response_time)
+        )
+        proxies = list(proxies)
+        current_timestamp = time.time()
+
+        return {
+            "proxies": [{
+                "address": proxy.address,
+                "response_time": proxy.response_time / 1000 if proxy.response_time is not None else None,
+                "uptime": datetime.timedelta(
+                    seconds=int(current_timestamp - proxy.uptime)) if proxy.uptime is not None else None,
+                "bad_uptime": datetime.timedelta(
+                    seconds=int(current_timestamp - proxy.bad_uptime)) if proxy.bad_uptime is not None else None,
+                "last_check_time": proxy.last_check_time,
+                "checking_period": proxy.checking_period,
+                "number_of_bad_checks": proxy.number_of_bad_checks,
+                "bad_proxy": proxy.bad_proxy,
+                "white_ipv4": proxy.white_ipv4,
+                "city": proxy.city,
+                "region": proxy.region,
+                "country_code": proxy.country_code,
+            } for proxy in proxies]
+        }
+
+    @get_response_wrapper("proxy_count_items.html")
+    async def get_proxy_count_items_html(self, request):
+        return {
+            "proxy_count_items": list(await db.execute(ProxyCountItem.select().order_by(ProxyCountItem.timestamp)))
+        }
+
+    async def get_best_http_proxy(self, request):
+        proxy_address = (await db.get(
+            Proxy.select().where(
+                Proxy.number_of_bad_checks == 0,
+                Proxy.raw_protocol == Proxy.PROTOCOLS.index("http"),
+            ).order_by(Proxy.response_time)
+        )).address
+
+        return web.Response(text=proxy_address)