new collector

Author: DevAlone

async_requests.py

@@ -59,6 +59,7 @@ def __str__(self):
 
 
 def get_random_user_agent():
-    return 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0'
+    return 'Mozilla/5.0 (Windows NT;) Gecko/20100101 Firefox/58.0'
+    # return 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0'
     # TODO: do it
     # return UserAgent().random

collectors/collector.py

@@ -1,3 +1,5 @@
+# TODO: add wrapper for doing requests and saving its cookies and UserAgent
+
 class AbstractCollector:
     # this method should return proxies in any of the following formats:
     # ip:port

collectors/freeproxylists_net/fake_data

@@ -0,0 +1,61 @@
+from collectors.pages_collector import PagesCollector
+
+import async_requests
+import re
+import lxml
+import lxml.html
+from urllib import parse
+
+
+BASE_URL = "http://freeproxylists.net/?page={}"
+
+
+class Collector(PagesCollector):
+    __collector__
+    # pages from 1
+
+    def __init__(self):
+        self.dynamic_pages_count = True
+
+    async def process_page(self, page_index):
+        # TODO: fix it
+        result = []
+        # data = {
+        #     "recaptcha_challenge_field": "03AMPJSYUZslP6_QOT6YguO3_jhhi2HJd81u0WHPe6eWrbMIStFeXQkfFV16GGQQuHZ_Za5HKMPKYTpPfLu-K5I8G8EAzbM8OTihtneSzgolnQB47SAhg5xw5sEHaTQj_B1I8u3gnw-Ts9ng3T6UAgi4jKTteAzoqQ4_DyND2DqWOX2kAVjCJKhLEYVZdp6z3-Qi14CS2PXRegNRALrYkBeALq5S3tY51y0A",
+        #     "recaptcha_response_field": "RPETS+ATTENTION",
+        # }
+        # resp = await async_requests.post(BASE_URL.format(page_index + 1), data)
+
+        headers = {
+            "Cookie": "hl=en; pv=14; userno=20180103-012864; from=direct; visited=2018%2F01%2F03+19%3A35%3A50; __utmb=251962462.16.10.1514975753; __utmc=251962462; __utmt=1; __atuvc=15%7C1; __atuvs=5a4cb2077864569600e; __utma=251962462.2036678170.1513732296.1513732296.1514975753.2; __utmz=251962462.1513732296.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmv=251962462.Ukraine"
+        }
+
+        resp = await async_requests.get(BASE_URL.format(page_index + 1), headers=headers)
+        text = resp.text
+        # with open("/home/user/python_projects/proxy_py/collectors/freeproxylists_net/fake_data") as f:
+        #     text = f.read()
+        try:
+            tree = lxml.html.fromstring(text)
+            table_element = tree.xpath(".//table[@class='DataGrid']")[0]
+        except BaseException:
+            raise Exception("table not found: {}".format(text))
+
+        rows = table_element.xpath('.//tr')
+        for row in rows:
+            try:
+                ip = row.xpath('.//td/script')[0].text
+                port = row.xpath('.//td')[1].text
+                if ip is None or port is None:
+                    continue
+
+                ip = parse.unquote(ip)
+                ip = re.search(r">([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}).*?</a>", ip)
+                if ip is None:
+                    continue
+                ip = ip.groups()[0]
+
+                result.append("{}:{}".format(ip, port))
+            except IndexError:
+                pass
+
+        return result

collectors/freeproxylists_net/freeproxylists_net.py

@@ -0,0 +1,36 @@
+from collectors.pages_collector import PagesCollector
+
+import async_requests
+import json
+
+
+URL_PATTERN = "https://nordvpn.com/wp-admin/admin-ajax.php?searchParameters[0][name]=proxy-country" \
+      "&searchParameters[0][value]=&searchParameters[1][name]=proxy-ports&searchParameters[1][value]=" \
+      "&offset={}&limit={}&action=getProxies"
+
+
+class Collector(PagesCollector):
+    # this collector gives a lot of bad proxies
+    __collector__ = False
+    processing_period = 10 * 60
+
+    def __init__(self):
+        self.pages_count = 10
+        self.limit = 100
+
+    async def process_page(self, page_index):
+        offset = page_index * self.limit
+        resp = await async_requests.get(URL_PATTERN.format(offset, self.limit))
+        json_response = json.loads(resp.text)
+
+        result = []
+
+        for item in json_response:
+            result.append("{}:{}".format(item["ip"], item["port"]))
+
+        if result:
+            self.pages_count = page_index + 2
+        else:
+            self.pages_count = page_index
+
+        return result

collectors/nordvpn_com/nordvpn_com.py

@@ -5,6 +5,10 @@
 class PagesCollector(AbstractCollector):
     async def collect(self):
         proxies = await self.process_page(self.current_page)
+
+        if self.dynamic_pages_count:
+            self.pages_count = self.current_page + 2 if proxies else self.current_page + 1
+
         self.current_page += 1
         if self.current_page >= self.pages_count:
             self.current_page = 0
@@ -20,5 +24,7 @@ async def process_page(self, page_index):
     # and also you should set pages_count
     pages_count = 0
     current_page = 0
+    # or set dynamic pages count
+    dynamic_pages_count = False
 
     processing_period = 60 * 10

collectors/pages_collector.py

@@ -15,14 +15,14 @@
 
 
 # TODO: add multiple checks with several sites
-async def check_proxy(proxy_url: str, session=None):
+async def check_proxy(proxy_url: str, session=None, timeout=None):
     try:
         res = await _request(
             'get',
             'https://pikagraphs.d3d.info/OK/',
             # 'https://wtfismyip.com/text',
             proxy_url,
-            settings.PROXY_CHECKING_TIMEOUT,
+            settings.PROXY_CHECKING_TIMEOUT if timeout is None else timeout,
             session
         )
 

proxy_utils.py

@@ -0,0 +1,50 @@
+from collectors.freeproxylists_net.freeproxylists_net import Collector
+
+import proxy_utils
+
+import asyncio
+import sys
+
+
+loop = asyncio.get_event_loop()
+
+
+async def check_raw_proxy(raw_proxy: str):
+    protocols = [
+        "http", "socks4", "socks5"
+    ]
+    works = False
+    for protocol in protocols:
+        # print("checking... {}".format(raw_proxy))
+        result = await proxy_utils.check_proxy("{}://{}".format(protocol, raw_proxy), timeout=5)
+        if result:
+            works = True
+            break
+
+    print("1" if works else "0", end="")
+    sys.stdout.flush()
+
+    return works
+
+
+async def main():
+    collector = Collector()
+    while True:
+        proxies = await collector.collect()
+        # print(proxies)
+        tasks = []
+        # print(proxies)
+        for proxy in proxies:
+            tasks.append(check_raw_proxy(proxy))
+
+        if tasks:
+            await asyncio.wait(tasks)
+        else:
+            print("Empty")
+        print()
+
+        await asyncio.sleep(5)
+
+
+if __name__ == '__main__':
+    loop.run_until_complete(main())