returning random proxies

Author: DevAlone

models.py

@@ -2,16 +2,20 @@
 import peewee
 import peewee_async
 
-db = peewee_async.PooledPostgresqlDatabase(
+raw_db = peewee_async.PooledPostgresqlDatabase(
     *settings.DATABASE_CONNECTION_ARGS,
     **settings.DATABASE_CONNECTION_KWARGS,
 )
 
+# TODO: make it work
+# raw_db.execute_sql('CREATE EXTENSION IF NOT EXISTS tsm_system_rows;')
+raw_db.execute_sql('''CREATE MATERIALIZED VIEW IF NOT EXISTS working_proxies 
+AS SELECT * FROM proxies WHERE number_of_bad_checks = 0;''')
 
 class Proxy(peewee.Model):
     class Meta:
-        database = db
-        db_table = 'proxies'
+        database = raw_db
+        raw_db_table = 'proxies'
         indexes = (
             (('raw_protocol', 'auth_data', 'domain', 'port'), True),
             (('auth_data', 'domain', 'port'), False),  # important!
@@ -109,8 +113,8 @@ def __str__(self):
 
 class ProxyCountItem(peewee.Model):
     class Meta:
-        database = db
-        db_table = 'proxy_count_items'
+        database = raw_db
+        raw_db_table = 'proxy_count_items'
 
     timestamp = peewee.IntegerField(primary_key=True)
     good_proxies_count = peewee.IntegerField(null=False)
@@ -120,8 +124,8 @@ class Meta:
 
 class CollectorState(peewee.Model):
     class Meta:
-        database = db
-        db_table = 'collector_states'
+        database = raw_db
+        raw_db_table = 'collector_states'
         indexes = (
             (('processing_period',), False),
             (('last_processing_time',), False),
@@ -139,14 +143,14 @@ class Meta:
 
 class StatBaseModel(peewee.Model):
     class Meta:
-        database = db
+        database = raw_db
 
     timestamp = peewee.BigIntegerField(primary_key=True)
 
 
 class NumberOfProxiesToProcess(StatBaseModel):
     class Meta:
-        db_table = 'number_of_proxies_to_process'
+        raw_db_table = 'number_of_proxies_to_process'
 
     good_proxies = peewee.IntegerField(null=False)
     bad_proxies = peewee.IntegerField(null=False)
@@ -155,14 +159,14 @@ class Meta:
 
 class NumberOfCollectorsToProcess(StatBaseModel):
     class Meta:
-        db_table = 'number_of_collectors_to_process'
+        raw_db_table = 'number_of_collectors_to_process'
 
     value = peewee.IntegerField(null=False)
 
 
 class ProcessorProxiesQueueSize(StatBaseModel):
     class Meta:
-        db_table = 'processor_proxies_queue_size'
+        raw_db_table = 'processor_proxies_queue_size'
 
     value = peewee.IntegerField(null=False)
 
@@ -175,4 +179,4 @@ class Meta:
 NumberOfCollectorsToProcess.create_table(_silent)
 ProcessorProxiesQueueSize.create_table(_silent)
 
-db = peewee_async.Manager(db)
+db = peewee_async.Manager(raw_db)

proxy_py/_settings.py

@@ -81,7 +81,7 @@
 PROXY_PROVIDER_SERVER_MAXIMUM_REQUEST_LENGTH = 1024
 PROXY_PROVIDER_SERVER_MAXIMUM_STRING_FIELD_SIZE = 128
 
-_PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG = {
+PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG = {
     'fields': [
         'address', 'protocol', 'auth_data', 'domain', 'port', 'last_check_time', 'number_of_bad_checks',
         'bad_proxy', 'uptime', 'response_time', 'white_ipv4', 'white_ipv6', 'city', 'country_code', 'region'
@@ -97,8 +97,8 @@
     'proxy': {
         'model_class': ['models', 'Proxy'],
         'methods': {
-            'get': _PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG,
-            'count': _PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG,
+            'get': PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG,
+            'count': PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG,
         }
     }
 }

server/api_v2/api_request_handler.py

@@ -1,5 +1,8 @@
 import re
 
+from peewee import RawQuery
+
+from models import Proxy, db
 from proxy_py import settings
 from server.api_v1.requests_to_models.request_parser import ParseError
 from server.base_app import BaseApp
@@ -12,6 +15,8 @@ def __init__(self, app: BaseApp):
         self.app = app
         self.methods = {
             'get_model': self.get_model,
+            'get_proxies_for_id': self.get_proxies_for_id,
+            'get_proxy_for_id': self.get_proxy_for_id,
         }
 
     async def handle(self, request: aiohttp.ClientRequest, method_name: str, post_data: dict):
@@ -79,8 +84,8 @@ async def handle(self, request: aiohttp.ClientRequest, method_name: str, post_da
         return response
 
     async def get_model(self, data: dict) -> dict:
-        if 'name' not in data:
-            raise ParseError('please, specify the "name" key')
+        # TODO: implement
+        validate_dict_must_have_key(data, 'name')
 
         model_name = data['name']
         validate_letters_digits_undescores(model_name)
@@ -92,6 +97,43 @@ async def get_model(self, data: dict) -> dict:
             "result": "model " + model_name,
         }
 
+    async def get_proxy_for_id(self, data: dict) -> dict:
+        data['number'] = 1
+        res = await self.get_proxies_for_id(data)
+        res["result"] = res["results"][0]
+        del res["results"]
+        del res["number_of_results"]
+        return res
+
+    async def get_proxies_for_id(self, data: dict) -> dict:
+        validate_dict_must_have_key(data, 'id')
+        validate_dict_must_have_key(data, 'number')
+        number = int(data['number'])
+        validate_uint(number)
+
+        # TODO: validate id
+        results = []
+
+        for item in await db.execute(Proxy.raw(
+                f'SELECT * FROM working_proxies TABLESAMPLE SYSTEM_ROWS({number});'
+        )):
+            obj = {}
+
+            for field_name in settings.PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG['fields']:
+                obj[field_name] = getattr(item, field_name)
+
+            results.append(obj)
+
+        return {
+            "number_of_results": len(results),
+            "results": results,
+        }
+
+
+def validate_dict_must_have_key(data: dict, key: str):
+    if key not in data:
+        raise ParseError(f'please, specify the "{key}" key')
+
 
 def validate_letters_digits_undescores(value):
     if len(value) > settings.PROXY_PROVIDER_SERVER_MAXIMUM_STRING_FIELD_SIZE:
@@ -106,3 +148,10 @@ def validate_regex(value: str, regex: str):
 
     if not re.match(regex, value):
         raise ParseError(f"value \"{value}\" doesn't match to regex \"{regex}\"")
+
+
+def validate_uint(value):
+    if type(value) is not int:
+        raise ParseError("value should be integer")
+    if value < 0:
+        raise ParseError("value should be positive")