Migrate to unified data storage and generation script

README.md

@@ -1,4 +1,44 @@
-ffdns
-=====
+Freifunk DNS repository
+=======================
 
-Config snippets to resolve TLDs from freifunk and other networks
+This repository is supposed to make the management of DNS in the ICVPN easier. It consists of two parts:
+
+Data
+----
+In the data directory, there are files containing information how to resolve the TLDs and other domains (think RDNS) of various Freifunk communities and other networks (like DN42 for example). Please send pull requests to insert/update your own data!
+
+Each community/network has a file of its own in a format as simple as
+```
+domain=ffhh
+domain=112.10.in-addr.arpa
+domain=d.0.d.f.2.b.b.2.1.5.d.f.ip6.arpa
+server=fd51:2bb2:fd0d::101
+server=fd51:2bb2:fd0d::e01
+server=10.112.1.1
+server=10.112.14.1
+```
+As you can see, there are lines starting with `domain=` which define domains "owned" by this community, and lines starting with `server=` defining the servers of this community that are able to resolve the domains.
+
+The format is intended to retype as few data as possbile.
+
+Generation script
+-----------------
+The Python script genconfig.py can generate configs for different DNS resolvers from the above data format. It currently supports bind9 (types `static-stub` and `forward`) and dnsmasq.
+
+It is capable of excluding some files from the data directory, so that you can exclude your own community (because you probably are the master for your own domains) and recursively resolve all others. You may also filter out IPv4 or IPv6 servers if you are operating a single-stack network.
+
+The complete help message reads:
+```
+Usage: genconfig.py [options]
+
+Options:
+  -h, --help            show this help message and exit
+  -f FMT, --format=FMT  Create config in format FMT. Possible values: bind,
+                        dnsmasq, bind-forward. Default: dnsmasq
+  -s DIR, --sourcedir=DIR
+                        Use files in DIR as input files. Default: data/
+  -x FILES, --exclude=FILES
+                        Exclude the comma-separated list of FILES in the
+                        sourcedir from the generation
+  --filter=FILTER       Only include certain servers. Possible choices: v4, v6
+```

bind/dn42.v4

@@ -1,14 +0,0 @@
-# config as of https://dn42.net/Services-DNS-Configuration
-
-zone "dn42" {
-  type forward;
-  forwarders { 172.22.0.53; };
-};
-zone "22.172.in-addr.arpa" {
-  type forward;
-  forwarders { 172.22.0.53; };
-};
-zone "23.172.in-addr.arpa" {
-  type forward;
-  forwarders { 172.22.0.53; };
-};

bind/ffhh

@@ -1,17 +0,0 @@
-# ffhh
-zone "ffhh" {
-  type static-stub;      
-  server-addresses { fd51:2bb2:fd0d::101; fd51:2bb2:fd0d::e01; };      
-};
-
-# 10.112.0.0/16 reverse
-zone "112.10.in-addr.arpa" {
-  type static-stub;
-  server-addresses { fd51:2bb2:fd0d::101; fd51:2bb2:fd0d::e01; };
-};
-
-# FD51:2BB2:FD0D::/48 reverse
-zone "d.0.d.f.2.b.b.2.1.5.d.f.ip6.arpa" {
-  type static-stub;
-  server-addresses { fd51:2bb2:fd0d::101; fd51:2bb2:fd0d::e01; };
-};

bind/ffhh.v4

@@ -1,17 +0,0 @@
-# ffhh
-zone "ffhh" {
-  type static-stub;      
-  server-addresses { 10.112.1.1; 10.112.14.1; };      
-};
-
-# 10.112.0.0/16 reverse
-zone "112.10.in-addr.arpa" {
-  type static-stub;
-  server-addresses { 10.112.1.1; 10.112.14.1; };
-};
-
-# FD51:2BB2:FD0D::/48 reverse
-zone "d.0.d.f.2.b.b.2.1.5.d.f.ip6.arpa" {
-  type static-stub;
-  server-addresses { 10.112.1.1; 10.112.14.1; };
-};

bind/hack.v4

@@ -1,32 +0,0 @@
-# config as of http://wiki.hamburg.ccc.de/ChaosVPN:DNS
-# only works if host is directly in chaosvpn?
-
-zone "hack" {
-  type static-stub;      
-  server-addresses { 172.31.0.5; };      
-};
-
-zone "31.172.in-addr.arpa" {
-  type static-stub;      
-  server-addresses { 172.31.0.5; };
-};
-
-zone "100.10.in-addr.arpa" {
-  type static-stub;      
-  server-addresses { 172.31.0.5; };
-};
-
-zone "101.10.in-addr.arpa" {
-  type static-stub;      
-  server-addresses { 172.31.0.5; };
-};
-
-zone "102.10.in-addr.arpa" {
-  type static-stub;      
-  server-addresses { 172.31.0.5; };
-};
-
-zone "103.10.in-addr.arpa" {
-  type static-stub;      
-  server-addresses { 172.31.0.5; };
-};

data/dn42

@@ -0,0 +1,7 @@
+# config as of https://dn42.net/Services-DNS-Configuration
+
+domain=dn42
+domain=22.172.in-addr.arpa
+domain=23.172.in-addr.arpa
+
+server=172.22.0.53

data/ffhb

@@ -0,0 +1,6 @@
+domain=ffhb
+domain=196.10.in-addr.arpa
+domain=c.2.f.0.9.1.1.5.f.2.d.f.ip6.arpa
+
+server=fd2f:5119:f2c::1
+server=10.196.0.1

data/ffhh

@@ -0,0 +1,8 @@
+domain=ffhh
+domain=112.10.in-addr.arpa
+domain=d.0.d.f.2.b.b.2.1.5.d.f.ip6.arpa
+
+server=fd51:2bb2:fd0d::101
+server=fd51:2bb2:fd0d::e01
+server=10.112.1.1
+server=10.112.14.1

data/hack

@@ -0,0 +1,11 @@
+# config as of http://wiki.hamburg.ccc.de/ChaosVPN:DNS
+# only works if host is directly in chaosvpn?
+
+domain=hack
+domain=31.172.in-addr.arpa
+domain=100.10.in-addr.arpa
+domain=101.10.in-addr.arpa
+domain=102.10.in-addr.arpa
+domain=103.10.in-addr.arpa
+
+server=172.31.0.5

dnsmasq/dn42.v4

@@ -1,5 +0,0 @@
-# config as of https://dn42.net/Services-DNS-Configuration
-
-server=/dn42/172.22.0.53
-server=/22.172.in-addr.arpa/172.22.0.53
-server=/23.172.in-addr.arpa/172.22.0.53

dnsmasq/ffhh

@@ -1,11 +0,0 @@
-# ffhh
-server=/ffhh/fd51:2bb2:fd0d::101
-server=/ffhh/fd51:2bb2:fd0d::e01
-
-# 10.112.0.0/16 reverse
-server=/112.10.in-addr.arpa/fd51:2bb2:fd0d::101
-server=/112.10.in-addr.arpa/fd51:2bb2:fd0d::e01
-
-# FD51:2BB2:FD0D::/48 reverse
-server=/d.0.d.f.2.b.b.2.1.5.d.f.ip6.arpa/fd51:2bb2:fd0d::101
-server=/d.0.d.f.2.b.b.2.1.5.d.f.ip6.arpa/fd51:2bb2:fd0d::e01

dnsmasq/ffhh.v4

@@ -1,11 +0,0 @@
-# ffhh
-server=/ffhh/10.112.1.1
-server=/ffhh/10.112.14.1
-
-# 10.112.0.0/16 reverse
-server=/112.10.in-addr.arpa/10.112.1.1
-server=/112.10.in-addr.arpa/10.112.14.1
-
-# FD51:2BB2:FD0D::/48 reverse
-server=/d.0.d.f.2.b.b.2.1.5.d.f.ip6.arpa/10.112.1.1
-server=/d.0.d.f.2.b.b.2.1.5.d.f.ip6.arpa/10.112.14.1

dnsmasq/hack.v4

@@ -1,9 +0,0 @@
-# config as of http://wiki.hamburg.ccc.de/ChaosVPN:DNS
-# only works if host is directly in chaosvpn?
-
-server=/hack/172.31.0.5
-server=/31.172.in-addr.arpa/172.31.0.5
-server=/100.10.in-addr.arpa/172.31.0.5
-server=/101.10.in-addr.arpa/172.31.0.5
-server=/102.10.in-addr.arpa/172.31.0.5
-server=/103.10.in-addr.arpa/172.31.0.5

genconfig.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+
+import os
+from textwrap import dedent
+from optparse import OptionParser
+from socket import AF_INET, AF_INET6, inet_pton
+
+class Formatter(object):
+    """
+    Abstract class to define the interface for formatters.
+    """
+
+    def __init__(self):
+        """
+        Initialize formatter, maybe generate a preamble for the config
+        """
+        self.config = []
+        self.add_comment(dedent(
+            """
+            This file is automatically generated by the ffdns generator.
+            Don't edit it manually! Instead, send pull requests to
+            https://github.com/freifunkhamburg/ffdns
+            and re-generate it using the genconfig.py there!
+            """
+        ))
+
+    def add_comment(self, comment):
+        """
+        Add a comment to the config. Default is prefixed with #
+        """
+        self.config.append("# " + "\n# ".join(comment.split("\n")))
+
+    def add_data(self, domains, servers):
+        """
+        Add config directives so that every domain in domains is forwarded to
+        every server in servers.
+        """
+        raise NotImplementedError()
+
+    def finalize(self):
+        """
+        Finalize configuration and return it
+        """
+        return "\n".join(self.config)
+
+class DnsmasqFormatter(Formatter):
+    "Formatter for dnsmasq"
+    def add_data(self, domains, servers):
+        for domain in domains:
+            for server in servers:
+                self.config.append("server=/%s/%s" % (domain, server))
+
+class BindFormatter(Formatter):
+    "Formatter for bind9 (>=9.8) using type static-stub"
+    def add_data(self, domains, servers):
+        for domain in domains:
+            self.config.append(dedent("""
+                zone "%s" {
+                    type static-stub;
+                    server-addresses { %s; };
+                };
+            """ % (domain, "; ".join(servers))).lstrip())
+
+class BindForwardFormatter(Formatter):
+    "Formatter for bind9 using type forward"
+    def add_data(self, domains, servers):
+        for domain in domains:
+            self.config.append(dedent("""
+                zone "%s" {
+                    type forward;
+                    forwarders { %s; };
+                    forward only;
+                };
+            """ % (domain, "; ".join(servers))).lstrip())
+
+def create_config(srcdir, fmtclass, exclude=None, filters=[]):
+    """
+    Generates a configuration using all files in srcdir (non-recursively)
+    except those in the iterable exclude.
+
+    The files are read in lexicographic order to produce deterministic results.
+
+    Every option=value pair in the files is passed through all callables in the
+    iterable filters. Only if none return False, it is assumed valid.
+    """
+    COMMENT_CHAR = "#"
+    OPTION_CHAR = "="
+    formatter = fmtclass()
+    for fname in sorted(list(set(os.listdir(srcdir)) - set(exclude))):
+        fpath = os.path.join(srcdir, fname)
+        if os.path.isfile(fpath):
+            domains = []
+            servers = []
+
+            with file(fpath) as f:
+                formatter.add_comment("\n%s\n" % fname)
+                for line in f:
+                    if COMMENT_CHAR in line:
+                        line, comment = line.split(COMMENT_CHAR, 1)
+                        comment = comment.strip()
+                        if comment:
+                            formatter.add_comment(comment)
+                    if OPTION_CHAR in line:
+                        option, value = line.split(OPTION_CHAR, 1)
+                        option = option.strip()
+                        value = value.strip()
+                        if not all(filt(option, value) for filt in filters):
+                            continue
+                        if option == "server":
+                            servers.append(value)
+                        elif option == "domain":
+                            domains.append(value)
+                        else:
+                            raise RuntimeError("Unknown option '%s' in file '%s'" % (option, fpath))
+                    elif line.strip():
+                        raise RuntimeError("Unrecognized line '%s' in file '%s'" % (line, fpath))
+
+            if not domains:
+                formatter.add_comment("No valid domains found")
+            elif not servers:
+                formatter.add_comment("No valid servers found")
+            else:
+                formatter.add_data(domains, servers)
+
+    print(formatter.finalize())
+
+if __name__ == "__main__":
+    def try_inet_pton(af, ip):
+        try:
+            inet_pton(af, ip)
+            return True
+        except:
+            return False
+
+    formatters = {
+        "dnsmasq": DnsmasqFormatter,
+        "bind": BindFormatter,
+        "bind-forward": BindForwardFormatter,
+    }
+    filters = {
+        "v4": lambda option, value: option != "server" or try_inet_pton(AF_INET, value),
+        "v6": lambda option, value: option != "server" or try_inet_pton(AF_INET6, value),
+    }
+    parser = OptionParser()
+    parser.add_option("-f", "--format", dest="fmt", help="Create config in format FMT. Possible values: %s. Default: dnsmasq" % ", ".join(formatters.keys()), metavar="FMT", choices=formatters.keys(), default="dnsmasq")
+    parser.add_option("-s", "--sourcedir", dest="src", help="Use files in DIR as input files. Default: data/", metavar="DIR", default="data")
+    parser.add_option("-x", "--exclude", dest="exclude", help="Exclude the comma-separated list of FILES in the sourcedir from the generation", metavar="FILES", default="")
+    parser.add_option("--filter", dest="filter", help="Only include certain servers. Possible choices: %s" % ", ".join(filters.keys()), choices=filters.keys())
+
+    (options, args) = parser.parse_args()
+
+    create_config(options.src, formatters[options.fmt], options.exclude.split(","), [filters[options.filter]] if options.filter else [])