Create package ffmap, add wiki input, remove old code

ffmap/inputs/alfred.py

@@ -0,0 +1,18 @@
+import subprocess
+import json
+
+class Input:
+    def __init__(self,request_data_type = 158):
+        self.request_data_type = request_data_type
+
+    def get_data(self, nodedb):
+        """Add data from alfred to the supplied nodedb"""
+        output = subprocess.check_output([
+            "alfred-json",
+            "-r", str(self.request_data_type),
+            "-f", "json",
+        ])
+        alfred_data = json.loads(output.decode("utf-8"))
+
+        for mac, node in alfred_data.items():
+            nodedb.add_or_update([mac], node)

ffmap/inputs/batadv.py

@@ -0,0 +1,100 @@
+import subprocess
+import json
+
+class Input:
+    """Fill the NodeDB with links from batadv-vis.
+
+    The links are added as lists containing the neighboring nodes, not
+    only their identifiers!  Mind this when exporting the database, as
+    it probably leads to recursion.
+    """
+    def __init__(self, mesh_interface="bat0"):
+        self.mesh_interface = mesh_interface
+
+    @staticmethod
+    def _is_similar_mac(a, b):
+        """Determine if two MAC addresses are similar."""
+        if a == b:
+            return True
+
+        # Split the address into bytes
+        try:
+            mac_a = list(int(i, 16) for i in a.split(":"))
+            mac_b = list(int(i, 16) for i in b.split(":"))
+        except ValueError:
+            return False
+
+        # Second and third byte musn't differ
+        if mac_a[1] != mac_b[1] or mac_a[2] != mac_b[2]:
+            return False
+
+        # First byte must only differ in bit 2
+        if mac_a[0] | 2 != mac_b[0] | 2:
+            return False
+
+        # Count differing bytes after the third
+        c = [x for x in zip(mac_a[3:], mac_b[3:]) if x[0] != x[1]]
+
+        # No more than two additional bytes must differ
+        if len(c) > 2:
+            return False
+
+        # If no more bytes differ, they are very similar
+        if len(c) == 0:
+            return True
+
+        # If the sum of absolute differences isn't greater than 2, they
+        # are pretty similar
+        delta = sum(abs(i[0] - i[1]) for i in c)
+        return delta < 2
+
+    def get_data(self, nodedb):
+        """Add data from batadv-vis to the supplied nodedb"""
+        output = subprocess.check_output([
+            "batadv-vis",
+            "-i", str(self.mesh_interface),
+            "-f", "jsondoc",
+        ])
+        data = json.loads(output.decode("utf-8"))
+
+        # First pass
+        for node in data["vis"]:
+            # Determine possible other MAC addresses of this node by
+            # comparing all its client's MAC addresses to its primary
+            # MAC address.  If they are similar, it probably is another
+            # address of the node itself!  If it isn't, it is a real
+            # client.
+            node['aliases'] = [node["primary"]]
+            if 'secondary' in node:
+                node['aliases'].extend(node['secondary'])
+            real_clients = []
+            for mac in node["clients"]:
+                if self._is_similar_mac(mac, node["primary"]):
+                    node['aliases'].append(mac)
+                else:
+                    real_clients.append(mac)
+            node['clients'] = real_clients
+
+            # Add nodes and aliases without any information at first.
+            # This way, we can later link the objects themselves.
+            nodedb.add_or_update(node['aliases'])
+
+        # Second pass
+        for node in data["vis"]:
+            # We only need the primary address now, all aliases are
+            # already present in the database.  Furthermore, we can be
+            # sure that all neighbors are in the database as well.  If
+            # a neighbor isn't added already, we simply ignore it.
+            nodedb.add_or_update(
+                [node["primary"]],
+                {
+                    "clients": node["clients"],
+                    "neighbors": [
+                        {
+                            "metric": neighbor['metric'],
+                            "neighbor": nodedb[neighbor['neighbor']],
+                        } for neighbor in node["neighbors"]
+                          if neighbor['neighbor'] in nodedb
+                    ]
+                }
+            )

ffmap/inputs/wiki.py

@@ -0,0 +1,71 @@
+import json
+import argparse
+from itertools import zip_longest
+from urllib.request import urlopen
+from bs4 import BeautifulSoup
+
+class Input:
+    def __init__(self, url="http://luebeck.freifunk.net/wiki/Knoten"):
+        self.url = url
+
+    def fetch_wikitable(self):
+        f = urlopen(self.url)
+        soup = BeautifulSoup(f)
+        table = soup.find("table")
+        rows = table.find_all("tr")
+        headers = []
+        data = []
+
+        def maybe_strip(x):
+            if isinstance(x.string, str):
+                return x.string.strip()
+            else:
+                return ""
+
+        for row in rows:
+            tds = list([maybe_strip(x) for x in row.find_all("td")])
+            ths = list([maybe_strip(x) for x in row.find_all("th")])
+
+            if any(tds):
+                data.append(tds)
+
+            if any(ths):
+                headers = ths
+
+        return [dict(zip(headers, d)) for d in data]
+
+    def get_data(self, nodedb):
+        nodes = self.fetch_wikitable()
+
+        for node in nodes:
+            if "MAC" not in node or not node["MAC"]:
+                # without MAC, we cannot merge this data with others, so
+                # we might as well ignore it
+                continue
+
+            newnode = {
+                "network": {
+                    "mac": node.get("MAC").lower(),
+                },
+                "location": {
+                    "latitude": float(node.get("GPS", " ").split(" ")[0]),
+                    "longitude": float(node.get("GPS", " ").split(" ")[1]),
+                    "description": node.get("Ort"),
+                } if " " in node.get("GPS", "") else None,
+                "hostname": node.get("Knotenname"),
+                "hardware": {
+                    "model": node["Router"],
+                } if node.get("Router") else None,
+                "software": {
+                    "firmware": {
+                        "base": "LFF",
+                        "release": node.get("LFF Version"),
+                    },
+                },
+                "owner": {
+                    "contact": node["Betreiber"],
+                } if node.get("Betreiber") else None,
+            }
+            # remove keys with None as value
+            newnode = {k: v for k,v in newnode.items() if v is not None}
+            nodedb.add_or_update([newnode["network"]["mac"]], newnode)