Add support for DNS

.gitlab-ci.yml

@@ -9,6 +9,7 @@ stages:
     - pip install --upgrade requests>=2.22.0
     - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium
     - pip install --upgrade -r requirements.txt
+    - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done
   script:
     - pip list
     - openssl version -a

.travis.yml

@@ -16,6 +16,7 @@ before_install:
   # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet
 install:
  - pip install --upgrade -r requirements.txt
+ - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done
  - pip list
 before_script:
   - openssl version -a

Dockerfile

@@ -1,28 +1,26 @@
 FROM alpine:3.8
 
-#Base settings
+# Base settings
 ENV HOME /root
+WORKDIR /root
 
-COPY requirements.txt /root/requirements.txt
+# Add ZeroNet source
+COPY . /root
+VOLUME /root/data
 
-#Install ZeroNet
+# Install dependencies
 RUN apk --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \
- && pip3 install -r /root/requirements.txt \
+ && pip3 install -r requirements.txt \
+ && for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip3 install -r ${PLUGIN}; done \
  && apk del python3-dev gcc libffi-dev musl-dev make \
  && echo "ControlPort 9051" >> /etc/tor/torrc \
  && echo "CookieAuthentication 1" >> /etc/tor/torrc
 
-#Add Zeronet source
-COPY . /root
-VOLUME /root/data
-
-#Control if Tor proxy is started
+# Control if Tor proxy is started
 ENV ENABLE_TOR false
 
-WORKDIR /root
-
-#Set upstart command
+# Set upstart command
 CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552
 
-#Expose ports
+# Expose ports
 EXPOSE 43110 26552

Vagrantfile

@@ -5,41 +5,43 @@ VAGRANTFILE_API_VERSION = "2"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  #Set box
-  config.vm.box = "ubuntu/trusty64"
+  # Set box
+  config.vm.box = "ubuntu/bionic64"
 
-  #Do not check fo updates
+  # Do not check fo updates
   config.vm.box_check_update = false
 
-  #Add private network
+  # Add private network
   config.vm.network "private_network", type: "dhcp"
 
-  #Redirect ports
+  # Redirect ports
   config.vm.network "forwarded_port", guest: 43110, host: 43110
   config.vm.network "forwarded_port", guest: 15441, host: 15441
 
-  #Sync folder using NFS if not windows
+  # Sync folder using NFS if not windows
   config.vm.synced_folder ".", "/vagrant",
       :nfs => !Vagrant::Util::Platform.windows?
 
-  #Virtal Box settings
+  # Virtal Box settings
   config.vm.provider "virtualbox" do |vb|
     # Don't boot with headless mode
-    #vb.gui = true
+    vb.gui = false
 
     # Set VM settings
     vb.customize ["modifyvm", :id, "--memory", "512"]
     vb.customize ["modifyvm", :id, "--cpus", 1]
   end
 
-  #Update system
+  # Update system
   config.vm.provision "shell",
       inline: "sudo apt-get update -y && sudo apt-get upgrade -y"
 
-  #Install deps
+  # Install dependencies
   config.vm.provision "shell",
-      inline: "sudo apt-get install msgpack-python python-gevent python-pip python-dev -y"
+      inline: "sudo apt-get install python3 python3-pip python3-dev gcc libffi-dev musl-dev make -y"
   config.vm.provision "shell",
-      inline: "sudo pip install msgpack --upgrade"
+      inline: "sudo pip3 install -r /vagrant/requirements.txt"
+  config.vm.provision "shell",
+      inline: "for PLUGIN in $(ls /vagrant/plugins/[^disabled-]*/requirements.txt); do sudo pip3 install -r ${PLUGIN}; done"
 
 end

plugins/DNS/ConfigPlugin.py

@@ -0,0 +1,21 @@
+from Plugin import PluginManager
+
+@PluginManager.registerTo('ConfigPlugin')
+class ConfigPlugin:
+    def createArguments(self):
+        nameservers = [
+            'https://doh.securedns.eu/dns-query',
+
+            'https://doh-de.blahdns.com/dns-query',
+            'https://doh-jp.blahdns.com/dns-query',
+            'https://doh-ch.blahdns.com/dns-query',
+
+            'https://doh.dnswarden.com/uncensored',
+        ]
+
+        group = self.parser.add_argument_group('DNS plugin')
+
+        group.add_argument('--dns_nameservers', help='Nameservers for DNS plugin', default=nameservers, metavar='address', nargs='*')
+        group.add_argument('--dns_configure', help='Configure resolver with system config for DNS plugin', action='store_true', default=False)
+
+        return super(ConfigPlugin, self).createArguments()

plugins/DNS/DNSResolver.py

@@ -0,0 +1,103 @@
+from Config import config
+
+import logging
+import time
+import json
+import re
+import os
+
+log = logging.getLogger('DNSPlugin')
+
+class DNSResolver:
+    loaded = False
+    cache = {}
+
+    def __init__(self, site_manager, nameservers, configure):
+        self.site_manager = site_manager
+        self.nameservers = nameservers
+        self.configure = configure
+
+    def load(self):
+        if not self.loaded:
+            self.loadModule()
+            self.loadCache()
+
+            self.resolver = dns.resolver.Resolver(configure=self.configure)
+
+            if not self.configure:
+                self.resolver.nameservers = self.nameservers
+
+            self.loaded = True
+
+    def loadModule(self):
+        import dns.resolver
+        import dnslink
+
+        if config.tor == 'always':
+            response = type('lamdbaobject', (object,), {})()
+            response.flags = dns.flags.TC
+
+            query = lambda *x, **y: response
+            dns.query.udp = query
+
+        globals()['dns'] = locals()['dns']
+        globals()['dnslink'] = locals()['dnslink']
+
+    def loadCache(self, path=os.path.join(config.data_dir, 'dns_cache.json')):
+        if os.path.isfile(path):
+            try: self.cache = json.load(open(path))
+            except json.decoder.JSONDecodeError: pass
+
+    def saveCache(self, path=os.path.join(config.data_dir, 'dns_cache.json')):
+        json.dump(self.cache, open(path, 'w'), indent=2)
+
+    def isDomain(self, address):
+        return re.match(r'(.*?)([A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)$', address)
+
+    def resolveDomain(self, domain):
+        if not self.loaded:
+            self.load()
+
+        domain = domain.lower()
+
+        cache_entry = self.cache.get(domain)
+        if cache_entry and time.time() < cache_entry['timeout']:
+            log.info('cache: %s -> %s', domain, cache_entry['address'])
+            return cache_entry['address']
+
+        try:
+            resolver_record = dnslink.resolve(domain, protocol='zeronet', resolver=self.resolver)[0]
+            resolver_entry = {'domain': domain, 'address': resolver_record.split('/', 2)[2]}
+        except IndexError:
+            resolver_entry = None
+
+        resolver_error = None
+        try:
+            self.resolver.query(domain, 'TXT')
+        except dns.resolver.NXDOMAIN:
+            pass
+        except dns.resolver.NoAnswer:
+            pass
+        except dns.exception.DNSException as err:
+            resolver_error = err
+
+        if resolver_entry and not resolver_error:
+            log.info('resolver: %s -> %s', domain, resolver_entry['address'])
+            self.saveInCache(resolver_entry)
+            return resolver_entry['address']
+
+        if cache_entry and resolver_error:
+            log.info('fallback: %s -> %s', domain, cache_entry['address'])
+            self.extendInCache(cache_entry)
+            return cache_entry['address']
+
+    def saveInCache(self, entry):
+        entry['timeout'] = time.time() + 60 * 60
+        self.cache[entry['domain']] = entry
+
+        self.saveCache()
+
+    def extendInCache(self, entry):
+        self.cache[entry['domain']]['timeout'] = time.time() + 60 * 15
+
+        self.saveCache()

plugins/DNS/SiteManagerPlugin.py

@@ -0,0 +1,34 @@
+from Config import config
+from Plugin import PluginManager
+
+from .DNSResolver import DNSResolver
+
+allow_reload = False
+
+@PluginManager.registerTo('SiteManager')
+class SiteManagerPlugin:
+    _dns_resolver = None
+
+    @property
+    def dns_resolver(self):
+        if not self._dns_resolver:
+            nameservers = config.dns_nameservers
+            configure = config.dns_configure
+
+            self._dns_resolver = DNSResolver(
+                site_manager=self,
+                nameservers=nameservers,
+                configure=configure,
+            )
+
+        return self._dns_resolver
+
+    def load(self, *args, **kwargs):
+        super(SiteManagerPlugin, self).load(*args, **kwargs)
+        self.dns_resolver.load()
+
+    def isDomain(self, address):
+        return self.dns_resolver.isDomain(address) or super(SiteManagerPlugin, self).isDomain(address)
+
+    def resolveDomain(self, domain):
+        return self.dns_resolver.resolveDomain(domain) or super(SiteManagerPlugin, self).resolveDomain(domain)

plugins/DNS/UiRequestPlugin.py

@@ -0,0 +1,23 @@
+from Plugin import PluginManager
+
+import re
+
+@PluginManager.registerTo('UiRequest')
+class UiRequestPlugin:
+    def __init__(self, *args, **kwargs):
+        from Site import SiteManager
+        self.site_manager = SiteManager.site_manager
+
+        super(UiRequestPlugin, self).__init__(*args, **kwargs)
+
+    def actionSiteMedia(self, path, **kwargs):
+        match = re.match(r'/media/(?P<address>[A-Za-z0-9-]+\.[A-Za-z0-9\.-]+)(?P<inner_path>/.*|$)', path)
+
+        if match:
+            domain = match.group('address')
+            address = self.site_manager.dns_resolver.resolveDomain(domain)
+
+            if address:
+                path = '/media/' + address + match.group('inner_path')
+
+        return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs)

plugins/DNS/__init__.py

@@ -0,0 +1,3 @@
+from . import ConfigPlugin
+from . import UiRequestPlugin
+from . import SiteManagerPlugin

plugins/DNS/plugin_info.json

@@ -0,0 +1,5 @@
+{
+	"name": "DNS",
+	"description": "Support classic DNS as domain system.",
+	"default": "enabled"
+}

plugins/DNS/requirements.txt

@@ -0,0 +1,2 @@
+dnspython @ git+https://github.com/filips123/dnspython@dns-over-https
+dnslink>=1.0.2,<2.0.0

plugins/Zeroname/plugin_info.json

@@ -0,0 +1,5 @@
+{
+	"name": "Zeroname",
+	"description": "Support Namecoin as domain system.",
+	"default": "enabled"
+}