little refactoring

Author: DevAlone

collectors/collector.py

@@ -12,6 +12,10 @@ class AbstractCollector:
     __collector__ = False
     """Set this variable to True in your collector's implementation"""
 
+    def __init__(self):
+        self.data = {}
+        self.saved_variables = set()
+
     async def collect(self):
         """
         This method should return proxies in any of the following formats:
@@ -62,6 +66,9 @@ async def load_state(self, state):
         self.processing_period = state.processing_period
         self.last_processing_proxies_count = state.last_processing_proxies_count
         self.data = json.loads(state.data) if state.data is not None and state.data else {}
+        if '_variables' in self.data:
+            for var_name in self.data['_variables']:
+                setattr(self, var_name, self.data['_variables'][var_name])
 
     async def save_state(self, state: models.CollectorState):
         """
@@ -71,11 +78,21 @@ async def save_state(self, state: models.CollectorState):
         state.last_processing_time = self.last_processing_time
         state.processing_period = self.processing_period
         state.last_processing_proxies_count = self.last_processing_proxies_count
+
+        for var_name in self.saved_variables:
+            if '_variables' not in self.data:
+                self.data['_variables'] = {}
+
+            self.data['_variables'][var_name] = getattr(self, var_name)
+
         state.data = json.dumps(self.data)
 
     last_processing_time = 0
     """time in unix timestamp(seconds from 01.01.1970)"""
 
+    last_processing_proxies_count = 0
+    """how many proxies we got on last request, do not change manually"""
+
     processing_period = 60 * 60
     """processing period in seconds"""
 
@@ -99,8 +116,7 @@ async def save_state(self, state: models.CollectorState):
     when some collector has requests time limit
     """
 
-    data = {}
-    # TODO: consider namespacing
+    data = None
     """
     here you can store some information,
     it will be written into and read from database
@@ -111,3 +127,9 @@ async def save_state(self, state: models.CollectorState):
     Don't use names starting with the underscore
     like this one: _last_page
     """
+
+    saved_variables = None
+    """
+    Set of variables which are saved to database automatically(inside data dict)
+    """
+

collectors/pages_collector.py

@@ -13,18 +13,12 @@ class PagesCollector(AbstractCollector):
     behavior, just set dynamic_pages_count to false and set pages_count manually.
     """
 
-    async def load_state(self, state):
-        await super(PagesCollector, self).load_state(state)
-        if "_current_page" in self.data:
-            self.current_page = self.data["_current_page"]
-
-        if "_pages_count" in self.data:
-            self.pages_count = self.data["_pages_count"]
-
-    async def save_state(self, state):
-        await super(PagesCollector, self).save_state(state)
-        self.data["_current_page"] = self.current_page
-        self.data["_pages_count"] = self.pages_count
+    def __init__(self):
+        super(PagesCollector, self).__init__()
+        self.last_proxies_list = []
+        self.saved_variables.add('current_page')
+        self.saved_variables.add('pages_count')
+        self.saved_variables.add('last_proxies_list')
 
     async def collect(self):
         proxies = list(
@@ -40,19 +34,17 @@ async def collect(self):
                 """
                 proxies_set = set(proxies)
 
-                if "_last_proxies_set" in self.data:
-                    if set(self.data["_last_proxies_set"]) == proxies_set:
-                        self.pages_count = self.current_page + 1
+                if set(self.last_proxies_list) == proxies_set:
+                    self.pages_count = self.current_page + 1
 
-                self.data["_last_proxies_set"] = list(proxies_set)
+                self.last_proxies_list = list(proxies_set)
             else:
                 self.pages_count = self.current_page + 1
 
         self.current_page += 1
         if self.current_page >= self.pages_count:
             self.current_page = 0
 
-
         return proxies
 
     async def process_page(self, page_index):
@@ -70,3 +62,5 @@ async def process_page(self, page_index):
     dynamic_pages_count = True
 
     processing_period = 60 * 10
+
+    last_proxies_list = None

collectors/web/com/gatherproxy/collector.py

@@ -11,6 +11,7 @@ class Collector(PagesCollector):
     __collector__ = True
 
     def __init__(self):
+        super(Collector, self).__init__()
         self.pages_count = 57
 
     async def process_page(self, page_index):

collectors/web/com/nordvpn/collector.py

@@ -16,6 +16,7 @@ class Collector(PagesCollector):
     processing_period = 10 * 60
 
     def __init__(self):
+        super(Collector, self).__init__()
         self.pages_count = 10
         self.limit = 100
 

collectors/web/com/premproxy/collector.py

@@ -9,6 +9,7 @@
 
 class BaseCollectorPremProxyCom(PagesCollector):
     def __init__(self, url, pages_count):
+        super(BaseCollectorPremProxyCom, self).__init__()
         self.url = url
         self.pages_count = pages_count
 
@@ -57,8 +58,8 @@ def __init__(self):
         super(Collector, self).__init__('https://premproxy.com/list/', 20)
 
 
-class Collector(BaseCollectorPremProxyCom):
+class CollectorSocksList(BaseCollectorPremProxyCom):
     __collector__ = True
 
     def __init__(self):
-        super(Collector, self).__init__('https://premproxy.com/socks-list/', 20)
+        super(CollectorSocksList, self).__init__('https://premproxy.com/socks-list/', 20)

collectors/web/net/checkerproxy/collector.py

@@ -9,6 +9,7 @@ class Collector(AbstractCollector):
     __collector__ = True
 
     def __init__(self):
+        super(Collector, self).__init__()
         self.processing_period = 3600 * 12
         self.time_delta = datetime.timedelta(-1)
 
@@ -27,5 +28,6 @@ class CollectorToday(Collector):
     __collector__ = True
 
     def __init__(self):
+        super(CollectorToday, self).__init__()
         self.processing_period = 3600 * 3
         self.time_delta = datetime.timedelta(0)

collectors/web/net/free_proxy_list/collector.py

@@ -7,6 +7,7 @@
 
 class BaseCollectorFreeProxyListNet(AbstractCollector):
     def __init__(self, url):
+        super(BaseCollectorFreeProxyListNet, self).__init__()
         self.url = url
 
     async def collect(self):

collectors/web/net/freeproxylists/collector.py

@@ -22,6 +22,7 @@ class Collector(PagesCollector):
     __collector__ = False
 
     def __init__(self):
+        super(Collector, self).__init__()
         self.dynamic_pages_count = True
 
     async def process_page(self, page_index):

collectors/web/org/proxy_list/proxy_list_org/collector.py renamed to collectors/web/org/proxy_list/collector.py

@@ -11,6 +11,7 @@ class Collector(PagesCollector):
     __collector__ = True
 
     def __init__(self):
+        super(Collector, self).__init__()
         self.pages_count = 10
 
     async def process_page(self, page_index):

models.py

@@ -132,6 +132,7 @@ class Meta:
     processing_period = peewee.IntegerField(null=False)
     last_processing_time = peewee.IntegerField(null=False)
     last_processing_proxies_count = peewee.IntegerField(default=0, null=False)
+    # TODO: add new proxies
     last_processing_new_proxies_count = peewee.IntegerField(default=0, null=False)
     data = peewee.TextField(default=None, null=True)