optimized db

Author: DevAlone

models.py

@@ -19,6 +19,7 @@ class Meta:
             (('domain',), False),
             (('port',), False),
             (('number_of_bad_checks',), False),
+            (('next_check_time',), False),
             (('last_check_time',), False),
             (('checking_period',), False),
             (('uptime',), False),
@@ -42,6 +43,7 @@ class Meta:
 
     checking_period = peewee.IntegerField(default=settings.MIN_PROXY_CHECKING_PERIOD, null=False)
     last_check_time = peewee.IntegerField(default=0, null=False)
+    next_check_time = peewee.IntegerField(default=0, null=False)
     number_of_bad_checks = peewee.IntegerField(default=0, null=False)
     uptime = peewee.IntegerField(default=None, null=True)
     bad_uptime = peewee.IntegerField(default=None, null=True)

processor.py

@@ -74,8 +74,8 @@ async def process_proxies(self):
                 proxies = await db.execute(
                     Proxy.select().where(
                         Proxy.number_of_bad_checks == 0,
-                        Proxy.last_check_time < time.time() - Proxy.checking_period,
-                    ).order_by(Proxy.last_check_time).limit(settings.NUMBER_OF_CONCURRENT_TASKS)
+                        Proxy.next_check_time < time.time(),
+                    ).order_by(Proxy.next_check_time).limit(settings.NUMBER_OF_CONCURRENT_TASKS)
                 )
                 if proxies:
                     self.good_proxies_are_processed = False
@@ -92,8 +92,8 @@ async def process_proxies(self):
                     Proxy.select().where(
                         Proxy.number_of_bad_checks > 0,
                         Proxy.number_of_bad_checks < settings.DEAD_PROXY_THRESHOLD,
-                        Proxy.last_check_time < time.time() - settings.BAD_PROXY_CHECKING_PERIOD,
-                    ).order_by(Proxy.last_check_time).limit(settings.NUMBER_OF_CONCURRENT_TASKS)
+                        Proxy.next_check_time < time.time(),
+                    ).order_by(Proxy.next_check_time).limit(settings.NUMBER_OF_CONCURRENT_TASKS)
                 )
 
                 await self.add_proxies_to_queue(proxies)
@@ -106,8 +106,8 @@ async def process_proxies(self):
                     Proxy.select().where(
                         Proxy.number_of_bad_checks >= settings.DEAD_PROXY_THRESHOLD,
                         Proxy.number_of_bad_checks < settings.DO_NOT_CHECK_ON_N_BAD_CHECKS,
-                        Proxy.last_check_time < time.time() - settings.DEAD_PROXY_CHECKING_PERIOD,
-                    ).order_by(Proxy.last_check_time).limit(settings.NUMBER_OF_CONCURRENT_TASKS)
+                        Proxy.next_check_time < time.time(),
+                    ).order_by(Proxy.next_check_time).limit(settings.NUMBER_OF_CONCURRENT_TASKS)
                 )
 
                 await self.add_proxies_to_queue(proxies)
@@ -293,6 +293,7 @@ async def process_proxy(self, raw_protocol: int, auth_data: str, domain: str, po
                     )
 
                     proxy.last_check_time = int(time.time())
+                    proxy.next_check_time = proxy.last_check_time + proxy.checking_period
                     proxy.number_of_bad_checks += 1
                     proxy.uptime = int(time.time())
 
@@ -374,4 +375,8 @@ async def create_or_update_proxy(
                 + (checking_time / settings.PROXY_CHECKING_TIMEOUT) \
                 * (settings.MAX_PROXY_CHECKING_PERIOD - settings.MIN_PROXY_CHECKING_PERIOD)
 
+            # TODO: bad and not working proxies period
+
+            proxy.next_check_time = proxy.last_check_time + proxy.checking_period
+
             await db.update(proxy)

proxy_py/_settings.py

@@ -83,7 +83,7 @@
 
 PROXY_PROVIDER_SERVER_API_CONFIG_FETCH_CONFIG = {
     'fields': [
-        'address', 'protocol', 'auth_data', 'domain', 'port', 'last_check_time', 'number_of_bad_checks',
+        'address', 'protocol', 'auth_data', 'domain', 'port', 'last_check_time', 'next_check_time', 'number_of_bad_checks',
         'bad_proxy', 'uptime', 'response_time', 'white_ipv4', 'white_ipv6', 'city', 'country_code', 'region'
     ],
     'filter_fields': [

statistics/statistics.py

@@ -55,23 +55,23 @@ async def number_of_proxies_to_process(timestamp):
     good_proxies_count = await db.count(
         Proxy.select().where(
             Proxy.number_of_bad_checks == 0,
-            Proxy.last_check_time < timestamp - Proxy.checking_period,
+            Proxy.next_check_time < timestamp - Proxy.checking_period,
         )
     )
 
     bad_proxies_count = await db.count(
         Proxy.select().where(
             Proxy.number_of_bad_checks > 0,
             Proxy.number_of_bad_checks < settings.DEAD_PROXY_THRESHOLD,
-            Proxy.last_check_time < timestamp - settings.BAD_PROXY_CHECKING_PERIOD,
+            Proxy.next_check_time < timestamp - settings.BAD_PROXY_CHECKING_PERIOD,
         )
     )
 
     dead_proxies_count = await db.count(
         Proxy.select().where(
             Proxy.number_of_bad_checks >= settings.DEAD_PROXY_THRESHOLD,
             Proxy.number_of_bad_checks < settings.DO_NOT_CHECK_ON_N_BAD_CHECKS,
-            Proxy.last_check_time < timestamp - settings.DEAD_PROXY_CHECKING_PERIOD,
+            Proxy.next_check_time < timestamp - settings.DEAD_PROXY_CHECKING_PERIOD,
         )
     )