add tcp sockets, update docs

Author: blind-oracle

README.md

@@ -11,7 +11,7 @@ This daemon was created to solve the problem of manipulating traffic based on do
 * Client sends DNS request to a recursive DNS server which supports DNSTap (**unbound**, **bind** etc)
 * DNS server logs the reply information through DNSTap to **dnstap-bgp**
 * **dnstap-bgp** caches the IPs from the reply and announces them through BGP
-* When the request for the already cached IP comes again - refresh it's TTL
+* When the request for the already cached IP comes again - refresh its TTL
 
 ## Features
 * Load a list of domains to intercept: the prefix tree is used to match subdomains
@@ -21,11 +21,11 @@ This daemon was created to solve the problem of manipulating traffic based on do
 * Export routes to any number of BGP peers
 * Configurable timeout to purge entries from the cache
 * Persist the cache on disk (in a Bolt database)
-* Sync the obtained IPs with other instances of **dnstap-bgp** using simple HTTP requests
+* Sync the obtained IPs with other instances of **dnstap-bgp**
 * Can switch itself to a pre-created network namespace before initializing network. This can be useful if you want to peer with a BGP server running on the same host (e.g. **bird** does not support peering with any of the local interfaces). This requires running as *root*.
 
 ## Synchronization
-**dnstap-bgp** can optionally push the obtained IPs to other **dnstap-bgp** instances. Also it periodically syncs its cache with peers to keep it up-to-date in case of network outages. The interaction is done using simple HTTP queries and JSON.
+**dnstap-bgp** can optionally push the obtained IPs to other **dnstap-bgp** instances. It also periodically syncs its cache with peers to keep it up-to-date in case of network outages. The interaction is done using simple HTTP queries and JSON.
 
 ## Limitations
 * IDN (punycode) domain names are currenly not supported and are silently skipped
@@ -38,17 +38,28 @@ This daemon was created to solve the problem of manipulating traffic based on do
 See *deploy/dnstap-bgp.toml* for an example configuration and description of parameters.
 
 ## Examples
-### unbound.conf
-```
-...
+DNSTap works in a client-server manner, where DNS server is *the client** and **dnstap-bgp** is a server.
+
+### Unbound
+Unbound seem to be able to work with DNSTap only through UNIX sockets.
 
+```
 dnstap:
     dnstap-enable: yes
     dnstap-socket-path: "/tmp/dnstap.sock"
-    dnstap-send-identity: no
-    dnstap-send-version: no
-
     dnstap-log-client-response-messages: yes
 ```
 
 **Important** In Ubuntu access to the DNSTap socket for Unbound is blocked by default by AppArmor rules. Either disable it for the Unbound binary or fix the rules.
+
+### BIND
+DNSTap is supported since 9.11, but usually is not built-in, at least in Ubuntu packages.
+BIND also can connect only using UNIX socket.
+
+```
+dnstap {
+    client response;
+};
+
+dnstap-output unix "/tmp/dnstap.sock"
+```

deploy/dnstap-bgp.toml

@@ -1,20 +1,24 @@
-# Path to a list of domains to match
+# Path to a list of domains to match - one domain per line
+# If a higher-level domain exists in the list - its subdomains will not be loaded, but still matched
 # Currently IDN domains are not supported
-domains = "/var/cache/rkn_domains.txt"
+domains = "/var/cache/domains.txt"
 
 # Path to a BoltDB file where to persist the cache
 # Optional
 cache = "/var/cache/dnstap-bgp.db"
 
 # TTL of the entries in cache
-# If the entry is not requested by clients for this duration the it's purged from the cache
+# If the entry is not requested by clients for this period then it's purged from the cache
+# Optional, default 24h
 ttl = "24h"
 
 [dnstap]
-# Path to a DNSTap socket
-socket = "/tmp/dnstap.sock"
+# IP:Port or a path to a UNIX socket file to listen on
+# listen = "0.0.0.0:1234"
+listen = "/tmp/dnstap.sock"
 
-# Permissions which are set on the socket file
+# Permissions which are set on the socket file if listening on UNIX socket
+# Optional, has no effect if using TCP
 perm = "0666"
 
 [bgp]
@@ -32,17 +36,24 @@ sourceIP = "192.168.113.1"
 # It it's also not defined - then RouterID
 nextHop = "192.168.112.1"
 
-# List of BGP peers
+# List of BGP peers in hostname or hostname:port formats
 peers = [
     "192.168.0.1",
     "192.168.0.2:177",
 ]
 
 [syncer]
-# Where to listen for sync requests
+# Where to listen for the sync requests
+# Optional, if not set - no incoming syncs will be allowed
 listen = "0.0.0.0:8080"
 
-# Sync peers
+# How frequently to perform full sync with peers
+# Optional, default 10m
+# If set to zero - no periodic syncs performed
+syncInterval = "10m"
+
+# Peers to sync with in hostname:port format
+# Optional, if not specified - no sync or push performed
 peers = [
     "192.168.0.2:8080",
 ]

dnstap.go

@@ -14,7 +14,7 @@ import (
 )
 
 type dnstapCfg struct {
-	Socket string
+	Listen string
 	Perm   string
 }
 
@@ -25,6 +25,8 @@ type dnstapServer struct {
 	cb          fCb
 	cbErr       fCbErr
 	fstrmServer *dnstap.FrameStreamSockInput
+	l           net.Listener
+	unixSocket  bool
 	ch          chan []byte
 }
 
@@ -101,22 +103,30 @@ func newDnstapServer(c *dnstapCfg, cb fCb, cbErr fCbErr) (ds *dnstapServer, err
 		cbErr: cbErr,
 	}
 
-	if c.Socket == "" {
-		log.Fatal("You need to specify DNSTap socket")
+	if c.Listen == "" {
+		return nil, fmt.Errorf("You need to specify DNSTap listening poing")
 	}
 
-	ds.fstrmServer, err = dnstap.NewFrameStreamSockInputFromPath(c.Socket)
-	if err != nil {
-		return nil, fmt.Errorf("DNSTap listening error: %s", err)
-	}
+	if addr, err := net.ResolveTCPAddr("tcp", c.Listen); err == nil {
+		if ds.l, err = net.ListenTCP("tcp", addr); err != nil {
+			return nil, fmt.Errorf("Unable to listen on '%s': %s", c.Listen, err)
+		}
 
-	if c.Perm != "" {
-		octal, err := strconv.ParseInt(c.Perm, 8, 32)
+		ds.fstrmServer = dnstap.NewFrameStreamSockInput(ds.l)
+	} else {
+		ds.fstrmServer, err = dnstap.NewFrameStreamSockInputFromPath(c.Listen)
 		if err != nil {
-			return nil, fmt.Errorf("Unable to parse '%s' as octal: %s", c.Perm, err)
+			return nil, fmt.Errorf("Unable to listen on '%s': %s", c.Listen, err)
 		}
 
-		os.Chmod(c.Socket, os.FileMode(octal))
+		if c.Perm != "" {
+			octal, err := strconv.ParseInt(c.Perm, 8, 32)
+			if err != nil {
+				return nil, fmt.Errorf("Unable to parse '%s' as octal: %s", c.Perm, err)
+			}
+
+			os.Chmod(c.Listen, os.FileMode(octal))
+		}
 	}
 
 	for i := 0; i < runtime.NumCPU(); i++ {

dnstap_test.go

@@ -30,7 +30,7 @@ func Test_DNSTap(t *testing.T) {
 	}
 
 	_, err := newDnstapServer(&dnstapCfg{
-		Socket: "dnstap.sock",
+		Listen: "dnstap.sock",
 		Perm:   "666",
 	}, cb, cbe)
 	assert.Nil(t, err)

main.go

@@ -193,7 +193,7 @@ func main() {
 		log.Fatalf("Unable to init DNSTap: %s", err)
 	}
 
-	log.Printf("Created DNSTap socket %s", cfg.DNSTap.Socket)
+	log.Printf("Listening for DNSTap on: %s", cfg.DNSTap.Listen)
 
 	go func() {
 		sigchannel := make(chan os.Signal, 1)

syncer.go

@@ -13,8 +13,9 @@ import (
 )
 
 type syncerCfg struct {
-	Listen string
-	Peers  []string
+	Listen       string
+	SyncInterval string
+	Peers        []string
 }
 
 type getAllFunc func() []*cacheEntry
@@ -25,7 +26,8 @@ type syncer struct {
 	s *http.Server
 	c *http.Client
 
-	peers []string
+	syncInterval time.Duration
+	peers        []string
 
 	getAll getAllFunc
 	add    addFunc
@@ -36,11 +38,18 @@ type syncer struct {
 
 func newSyncer(cf *syncerCfg, getAll getAllFunc, add addFunc, syncCb syncFunc) (s *syncer, err error) {
 	s = &syncer{
-		getAll:   getAll,
-		add:      add,
-		peers:    cf.Peers,
-		syncCb:   syncCb,
-		shutdown: make(chan struct{}),
+		getAll:       getAll,
+		add:          add,
+		peers:        cf.Peers,
+		syncCb:       syncCb,
+		shutdown:     make(chan struct{}),
+		syncInterval: 10 * time.Minute,
+	}
+
+	if cf.SyncInterval != "" {
+		if s.syncInterval, err = time.ParseDuration(cf.SyncInterval); err != nil {
+			return nil, fmt.Errorf("Unable to parse syncInterval: %s", err)
+		}
 	}
 
 	if len(cf.Peers) > 0 {
@@ -49,7 +58,9 @@ func newSyncer(cf *syncerCfg, getAll getAllFunc, add addFunc, syncCb syncFunc) (
 		}
 	}
 
-	go s.syncScheduler()
+	if s.syncInterval > 0 {
+		go s.syncScheduler()
+	}
 
 	if cf.Listen == "" {
 		return
@@ -133,12 +144,13 @@ func (s *syncer) callPeer(p, handler, method string, body io.ReadCloser) (resp *
 }
 
 func (s *syncer) syncScheduler() {
-	t := time.NewTicker(time.Minute)
+	t := time.NewTicker(s.syncInterval)
 
 	for {
 		select {
 		case <-t.C:
 			s.syncAll()
+
 		case <-s.shutdown:
 			return
 		}