ordering by multiple fields, limits and offsets, tests

Author: DevAlone

proxy_py/_settings.py

@@ -28,17 +28,21 @@
     'PORT': 55555,
 }
 
+_PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG = {
+    'fields': ['address', 'protocol', 'auth_data', 'domain', 'port', 'last_check_time',
+               'number_of_bad_checks', 'bad_proxy', 'uptime', 'response_time'],
+    'filter_fields': ['last_check_time', 'protocol', 'number_of_bad_checks', 'bad_proxy', 'uptime',
+                      'response_time'],
+    'order_by_fields': ['last_check_time', 'number_of_bad_checks', 'uptime', 'response_time'],
+    'default_order_by_fields': ['response_time', ],
+}
+
 PROXY_PROVIDER_SERVER_API_CONFIG = {
     'proxy': {
-        'modelClass': ('models', 'Proxy'),
+        'model_class': ('models', 'Proxy'),
         'methods': {
-            'get': {
-                'fields': ['address', 'protocol', 'auth_data', 'domain', 'port', 'last_check_time',
-                           'number_of_bad_checks', 'bad_proxy', 'uptime', 'response_time'],
-                'filterFields': ['last_check_time', 'protocol', 'number_of_bad_checks', 'bad_proxy', 'uptime',
-                                 'response_time'],
-                'orderFields': ['last_check_time', 'number_of_bad_checks', 'uptime'],
-            }
+            'get': _PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG,
+            'count': _PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG,
         }
     }
 }

server/requests_to_models/request.py

@@ -4,11 +4,22 @@ def __init__(self, class_name):
         self.class_name = class_name
 
 
-class GetRequest(Request):
-    def __init__(self, class_name, fields: list=None):
-        super(GetRequest, self).__init__(class_name)
+class FetchRequest(Request):
+    def __init__(self, class_name, fields: list=None, order_by: list=None):
+        super(FetchRequest, self).__init__(class_name)
         self.fields = fields if fields is not None else []
+        self.order_by = order_by if order_by is not None else []
+        self.limit = 0
+        self.offset = 0
 
+
+class GetRequest(FetchRequest):
     @staticmethod
     def from_request(request: Request):
         return GetRequest(request.class_name)
+
+
+class CountRequest(FetchRequest):
+    @staticmethod
+    def from_request(request: Request):
+        return CountRequest(request.class_name)

server/requests_to_models/request_executor.py

@@ -1,39 +1,64 @@
 from models import session
-from server.requests_to_models.request import Request, GetRequest
+from server.requests_to_models.request import Request, GetRequest, CountRequest, FetchRequest
 import importlib
 
 
 class RequestExecutor:
     def execute(self, request: Request):
         try:
-            return {
-                GetRequest: self._get,
-            }[type(request)](request)
+            if isinstance(request, FetchRequest):
+                return self._fetch(request)
+            # return {
+            #     # GetRequest: self._get,
+            #     # CountRequest: self._count
+            #     FetchRequest: self._fetch
+            # }[type(request)](request)
         except BaseException as ex:
             raise ExecutionError(repr(ex))
 
-    def _get(self, request: GetRequest):
+    def _fetch(self, request: FetchRequest):
         package = importlib.import_module(request.class_name[0])
         class_name = getattr(package, request.class_name[1])
 
         # TODO: remove bad_proxy
 
         queryset = session.query(class_name).filter(class_name.number_of_bad_checks == 0)
-        result = []
 
-        for item in queryset:
-            obj = {}
+        result = {
+            'count': queryset.count(),
+        }
 
-            for field_name in request.fields:
-                obj[field_name] = getattr(item, field_name)
+        if type(request) is GetRequest:
+            if request.order_by:
+                queryset = queryset.order_by(*self.order_by_list_to_sqlalchemy(request.order_by, class_name))
 
-            result.append(obj)
+            if request.limit > 0:
+                queryset = queryset.limit(request.limit)
 
-        return {
-            'data': result,
-            'count': queryset.count(),
-            'has_more': False,
-        }
+            if request.offset > 0:
+                queryset = queryset.offset(request.offset)
+
+            data = []
+
+            for item in queryset:
+                obj = {}
+
+                for field_name in request.fields:
+                    obj[field_name] = getattr(item, field_name)
+
+                data.append(obj)
+
+            result['data'] = data
+            result['has_more'] = request.offset + request.limit < result['count']
+
+        return result
+
+    def order_by_list_to_sqlalchemy(self, order_by_fields: list, class_name):
+        result = []
+        for field in order_by_fields:
+            result.append(getattr(class_name, field))
+
+        return result
 
 
 class ExecutionError(Exception):

server/requests_to_models/request_parser.py

@@ -1,4 +1,4 @@
-from server.requests_to_models.request import Request, GetRequest
+from server.requests_to_models.request import Request, GetRequest, CountRequest
 
 import string
 import copy
@@ -23,6 +23,11 @@ def parse(self, request: dict):
 
             if key in self.COMMA_SEPARATED_KEYS:
                 request[key] = self.comma_separated_field_to_list(request[key])
+            if key in {'limit', 'offset'}:
+                try:
+                    request[key] = int(request[key])
+                except ValueError:
+                    raise ValidationError('Value of key "{}" should be integer'.format(key))
 
             self.validate_value(key, request[key])
 
@@ -32,7 +37,7 @@ def validate_value(self, key: str, value):
         if type(value) not in [str, int, list]:
             raise ValidationError('Value type should be string, integer or list')
 
-        if len(value) > self.MAXIMUM_VALUE_LENGTH:
+        if type(value) in [str, list] and len(value) > self.MAXIMUM_VALUE_LENGTH:
             raise ValidationError(
                 'Some value is too big. Maximum allowed length is {}'.format(self.MAXIMUM_VALUE_LENGTH))
 
@@ -92,7 +97,7 @@ def parse_dict(self, req_dict):
 
         config = self.config[req_dict['model']]
 
-        result_request = Request(config['modelClass'])
+        result_request = Request(config['model_class'])
 
         if 'method' not in req_dict:
             raise ParseError('You should specify "method"')
@@ -105,26 +110,52 @@ def parse_dict(self, req_dict):
         config = config['methods'][method]
 
         return {
-            'get': self._get,
+            'get': self.method_get,
+            'count': self.method_count,
         }[method](req_dict, config, result_request)
 
-    def _get(self, req_dict, config, result_request):
-        fields = []
+    def method_get(self, req_dict, config, result_request):
+        result_request = GetRequest.from_request(result_request)
+        result_request.fields = self.parse_fields(req_dict, config)
+        result_request.order_by = self.parse_order_by_fields(req_dict, config)
+        if 'limit' in req_dict:
+            result_request.limit = req_dict['limit']
+        if 'offset' in req_dict:
+            result_request.offset = req_dict['offset']
 
-        if 'fields' not in req_dict:
-            fields = copy.copy(config['fields'])
-        else:
-            for field in req_dict['fields']:
-                if field not in config['fields']:
-                    raise ParseError("Field \"{}\" doesn't exist or isn't allowed".format(field))
+        return result_request
 
-                fields.append(field)
-
-        result_request = GetRequest.from_request(result_request)
-        result_request.fields = fields
+    def method_count(self, req_dict, config, result_request):
+        result_request = CountRequest.from_request(result_request)
+        result_request.fields = self.parse_fields(req_dict, config)
+        result_request.order_by = self.parse_order_by_fields(req_dict, config)
+        if 'limit' in req_dict:
+            result_request.limit = req_dict['limit']
+        if 'offset' in req_dict:
+            result_request.offset = req_dict['offset']
 
         return result_request
 
+    def parse_fields(self, req_dict, config):
+        return self.parse_list(req_dict, config, "fields", "fields", config['fields'])
+
+    def parse_order_by_fields(self, req_dict, config):
+        return self.parse_list(req_dict, config, "order_by", "order_by_fields", config['default_order_by_fields'])
+
+    def parse_list(self, req_dict, config, request_key, config_key, default_value):
+        if request_key not in req_dict:
+            return copy.copy(default_value)
+
+        result = []
+
+        for field in req_dict[request_key]:
+            if field not in config[config_key]:
+                raise ParseError("Field \"{}\" doesn't exist or isn't allowed".format(field))
+
+            result.append(field)
+
+        return result
+
     def _validate_config(self):
         # TODO: check fields for existence and so on
         if False:

test

@@ -0,0 +1,88 @@
+import asyncio
+import aiohttp
+import json
+
+
+API_URL = "http://localhost:55555"
+
+
+async def get_proxies(session, request):
+    async with session.post(API_URL, json=request) as resp:
+        json_data = json.loads(await resp.text())
+        return json_data['data']
+
+
+async def test_ordering(session, field_name):
+    request_data = {
+        'method': 'get',
+        'model': 'proxy',
+        'order_by': field_name
+    }
+
+    previous_proxy = None
+
+    for proxy in await get_proxies(session, request_data):
+        if previous_proxy is not None:
+            if previous_proxy[field_name] > proxy[field_name]:
+                return False
+
+        previous_proxy = proxy
+
+    return True
+
+
+async def test_complex_ordering(session, *args):
+    fields = args
+
+    request_data = {
+        'method': 'get',
+        'model': 'proxy',
+        'order_by': ', '.join(fields)
+    }
+
+    previous_proxy = None
+
+    for proxy in await get_proxies(session, request_data):
+        if previous_proxy is not None:
+            for i in range(1, len(fields)):
+                previous_field = fields[i - 1]
+                field = fields[i]
+
+                if previous_proxy[previous_field] > proxy[previous_field]:
+                    return False
+                elif previous_proxy[previous_field] < proxy[previous_field]:
+                    break
+
+        previous_proxy = proxy
+
+    return True
+
+
+async def run_tests(session):
+    tests = [
+        (test_ordering, 'response_time'),
+        (test_ordering, 'uptime'),
+        (test_ordering, 'number_of_bad_checks'),
+        (test_ordering, 'last_check_time'),
+        (test_complex_ordering, 'uptime', 'last_check_time'),
+        (test_complex_ordering, 'number_of_bad_checks', 'uptime', 'response_time'),
+    ]
+
+    for test in tests:
+        try:
+            result = await test[0](session, *test[1:])
+            print("PASSED" if result else "FAILED", end='')
+        except BaseException as ex:
+            print("FAILED DUE TO EXCEPTION: {}".format(ex), end='')
+        finally:
+            print(" test {} ".format(test))
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        await run_tests(session)
+
+
+if __name__ == '__main__':
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(main())