Support for "bat-ffhh" interface, add generate_aliases_v2.py

- Von nodelist.json auf nodes.json gewechselt
- nodes.json für Süd Domäne hinzugefügt
- Liest nodes.json's nun aus URLs, da sie auf verschiedenen servern liegen
- Durchsucht die nodes.json's nur noch als Strings, statt JSON auszuwerten

README.md

@@ -8,9 +8,6 @@ ffmap-backend gathers information on the batman network by invoking :
  * alfred-json and
  * batadv-vis
 
-In order to use alfred-json and batadv-vis make sure the user running this
-backend is allowed to access alfred's socket.
-
 The output will be written to a directory (`-d output`).
 
 Run `backend.py --help` for a quick overview of all available options.
@@ -18,9 +15,42 @@ Run `backend.py --help` for a quick overview of all available options.
 For the script's regular execution add the following to the crontab:
 
 <pre>
-* * * * * /path/to/ffmap-backend/backend.py -d /path/to/output -a /path/to/aliases.json --vpn ae:7f:58:7d:6c:2a --vpn d2:d0:93:63:f7:da
+* * * * * backend.py -d /path/to/output -a /path/to/aliases.json --vpn ae:7f:58:7d:6c:2a d2:d0:93:63:f7:da
 </pre>
 
+# Dependencies
+
+- Python 3
+- Python 3 Package [Networkx](https://networkx.github.io/)
+- [alfred-json](https://github.com/tcatm/alfred-json)
+- rrdtool (if run with `--with-rrd`)
+
+# Running as unprivileged user
+
+Some information collected by ffmap-backend requires access to specific system resources.
+
+Make sure the user you are running this under is part of the group that owns the alfred socket, so
+alfred-json can access the alfred daemon.
+
+    # ls -al /var/run/alfred.sock
+    srw-rw---- 1 root alfred 0 Mar 19 22:00 /var/run/alfred.sock=
+    # adduser map alfred
+    Adding user `map' to group `alfred' ...
+    Adding user map to group alfred
+    Done.
+    $ groups
+    map alfred
+
+Running batctl requires passwordless sudo access, because it needs to access the debugfs to retrive
+the gateway list.
+
+    # echo 'map ALL = NOPASSWD: /usr/sbin/batctl' | tee /etc/sudoers.d/map
+    map ALL = NOPASSWD: /usr/sbin/batctl
+    # chmod 0440 /etc/sudoers.d/map
+
+That should be everything. The script automatically detects if it is run in unprivileged mode and
+will prefix `sudo` where necessary.
+
 # Data format
 
 ## nodes.json
@@ -49,6 +79,21 @@ For the script's regular execution add the following to the crontab:
 - online
 - gateway
 
+## Old data format
+
+If you want to still use the old [ffmap-d3](https://github.com/ffnord/ffmap-d3)
+front end, you can use the file `ffmap-d3.jq` to convert the new output to the
+old one:
+
+```
+jq -n -f ffmap-d3.jq \
+    --argfile nodes nodedb/nodes.json \
+    --argfile graph nodedb/graph.json \
+    > nodedb/ffmap-d3.json
+```
+
+Then point your ffmap-d3 instance to the `ffmap-d3.json` file.
+
 # Removing owner information
 
 If you'd like to redact information about the node owner from `nodes.json`,

alfred_merge.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import subprocess
+import json
+
+from collections import MutableMapping
+
+def rec_merge(d1, d2):
+    '''
+    Update two dicts of dicts recursively, 
+    if either mapping has leaves that are non-dicts, 
+    the second's leaf overwrites the first's.
+    '''
+    for k, v in d1.items(): # in Python 2, use .iteritems()!
+        if k in d2:
+            # this next check is the only difference!
+            if all(isinstance(e, MutableMapping) for e in (v, d2[k])):
+                d2[k] = rec_merge(v, d2[k])
+            # we could further check types and merge as appropriate here.
+    d3 = d1.copy()
+    d3.update(d2)
+    return d3
+
+
+class alfred_merge:
+  def __init__(self,request_data_type_1 = 158, request_data_type_2 = 159):
+    self.request_data_type_1 = request_data_type_1
+    self.request_data_type_2 = request_data_type_2
+
+  def aliases(self):
+    output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_1),"-f","json"])
+    alfred_data_1 = json.loads(output.decode("utf-8"))
+    output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_2),"-f","json"])
+    alfred_data_2 = json.loads(output.decode("utf-8"))
+    
+    return json.dumps(rec_merge(alfred_data_1, alfred_data_2))
+
+
+if __name__ == "__main__":
+  ad = alfred_merge()
+  al = ad.aliases()
+  print(al)
+

aliases.json_sample

@@ -7,18 +7,30 @@
       "latitude": 53.86
     },
     "network": {
-      "mesh_interfaces": [
+      "mesh": {
-        "00:25:86:e6:f1:bf"
+        "bat0": {
-      ]
+          "interfaces": {
+            "tunnel": [
+              "00:25:86:e6:f1:bf"
+            ]
+          }
+        }
+      }
     }
   },
   {
     "node_id": "gw1",
     "hostname": "burgtor",
     "network": {
-      "mesh_interfaces": [
+      "mesh": {
-        "52:54:00:f3:62:d9"
+        "bat0": {
-      ]
+          "interfaces": {
+            "tunnel": [
+              "52:54:00:f3:62:d9"
+            ]
+          }
+        }
+      }
     }
   }
 ]

backend.py

@@ -16,6 +16,8 @@ from lib import graph, nodes
 from lib.alfred import Alfred
 from lib.batman import Batman
 from lib.rrddb import RRD
+from lib.nodelist import export_nodelist
+from lib.validate import validate_nodeinfos
 
 NODES_VERSION = 1
 GRAPH_VERSION = 1
@@ -25,7 +27,13 @@ def main(params):
     os.makedirs(params['dest_dir'], exist_ok=True)
 
     nodes_fn = os.path.join(params['dest_dir'], 'nodes.json')
+    tmp_nodes_fn = os.path.join(params['dest_dir'], 'nodes.json.tmp')
+
     graph_fn = os.path.join(params['dest_dir'], 'graph.json')
+    tmp_graph_fn = os.path.join(params['dest_dir'], 'graph.json.tmp')
+
+    nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json')
+    tmp_nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json.tmp')
 
     now = datetime.utcnow().replace(microsecond=0)
 
@@ -74,14 +82,16 @@ def main(params):
 
     # integrate alfred nodeinfo
     for alfred in alfred_instances:
-        nodes.import_nodeinfo(nodedb['nodes'], alfred.nodeinfo(),
+        nodeinfo = validate_nodeinfos(alfred.nodeinfo())
+        nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
                               now, assume_online=True)
 
     # integrate static aliases data
     for aliases in params['aliases']:
         with open(aliases, 'r') as f:
+#           nodeinfo = validate_nodeinfos(json.load(f))
             nodes.import_nodeinfo(nodedb['nodes'], json.load(f),
-                                  now, assume_online=False)
+                                  now, assume_online=False, statics=True)
 
     nodes.reset_statistics(nodedb['nodes'])
     for alfred in alfred_instances:
@@ -92,7 +102,6 @@ def main(params):
     for batman in batman_instances:
         vd = batman.vis_data()
         gwl = batman.gateway_list()
-
         mesh_info.append((vd, gwl))
 
     # update nodedb from batman-adv data
@@ -115,19 +124,41 @@ def main(params):
     if params['vpn']:
         graph.mark_vpn(batadv_graph, frozenset(params['vpn']))
 
+    def extract_tunnel(nodes):
+        macs = set()
+        for id, node in nodes.items():
+            try:
+                for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
+                    macs.add(mac)
+                for mac in node["nodeinfo"]["network"]["mesh"]["bat-ffhh"]["interfaces"]["tunnel"]:
+                    macs.add(mac)
+            except KeyError:
+                pass
+
+        return macs
+
+    graph.mark_vpn(batadv_graph, extract_tunnel(nodedb['nodes']))
+
     batadv_graph = graph.merge_nodes(batadv_graph)
     batadv_graph = graph.to_undirected(batadv_graph)
 
     # write processed data to dest dir
-    with open(nodes_fn, 'w') as f:
+    with open(tmp_nodes_fn, 'w') as f:
         json.dump(nodedb, f)
 
     graph_out = {'batadv': json_graph.node_link_data(batadv_graph),
                  'version': GRAPH_VERSION}
 
-    with open(graph_fn, 'w') as f:
+    with open(tmp_graph_fn, 'w') as f:
         json.dump(graph_out, f)
 
+    with open(tmp_nodelist_fn, 'w') as f:
+        json.dump(export_nodelist(now, nodedb), f)
+
+    os.rename(tmp_nodes_fn, nodes_fn)
+    os.rename(tmp_graph_fn, graph_fn)
+    os.rename(tmp_nodelist_fn, nodelist_fn)
+
     # optional rrd graphs (trigger with --rrd)
     if params['rrd']:
         script_directory = os.path.dirname(os.path.realpath(__file__))
@@ -136,14 +167,12 @@ def main(params):
         rrd.update_database(nodedb['nodes'])
         rrd.update_images()
 
-
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
 
     parser.add_argument('-a', '--aliases',
                         help='Read aliases from FILE',
-                        default=[], action='append',
+                        nargs='+', default=[], metavar='FILE')
-                        metavar='FILE')
     parser.add_argument('-m', '--mesh',
                         default=['bat0'], nargs='+',
                         help='Use given batman-adv mesh interface(s) (defaults'

ffmap-d3.jq

@@ -0,0 +1,52 @@
+{
+    "meta": {
+        "timestamp": $nodes.timestamp
+    },
+    "nodes": (
+        $graph.batadv.nodes
+        | map(
+            if has("node_id") and .node_id
+            then (
+                $nodes.nodes[.node_id] as $node
+                | {
+                    "id": .id,
+                    "uptime": $node.statistics.uptime,
+                    "flags": ($node.flags + {"client": false}),
+                    "name": $node.nodeinfo.hostname,
+                    "clientcount": (if $node.statistics.clients >= 0 then $node.statistics.clients else 0 end),
+                    "hardware": $node.nodeinfo.hardware.model,
+                    "firmware": $node.nodeinfo.software.firmware.release,
+                    "geo": (if $node.nodeinfo.location then [$node.nodeinfo.location.latitude, $node.nodeinfo.location.longitude] else null end),
+                    #"lastseen": $node.lastseen,
+                    "network": $node.nodeinfo.network
+                }
+            )
+            else
+                {
+                    "flags": {},
+                    "id": .id,
+                    "geo": null,
+                    "clientcount": 0
+                }
+            end
+        )
+    ),
+    "links": (
+        $graph.batadv.links
+        | map(
+            $graph.batadv.nodes[.source].node_id as $source_id
+            | $graph.batadv.nodes[.target].node_id as $target_id
+            | select(
+                $source_id and $target_id and
+                ($nodes.nodes | (has($source_id) and has($target_id)))
+            )
+            | {
+                "target": .target,
+                "source": .source,
+                "quality": "\(.tq), \(.tq)",
+                "id": ($source_id + "-" + $target_id),
+                "type": (if .vpn then "vpn" else null end)
+            }
+        )
+    )
+}

gateway.json

@@ -0,0 +1,29 @@
+[
+  {
+    "node_id": "deadbfff0101",
+    "hostname": "gw01"
+  },
+  {
+    "node_id": "deadbeef0505",
+    "hostname": "gw02.hamburg.freifunk.net",
+    "network": {
+        "mac": "de:ad:be:ef:05:05",
+        "mesh": {
+            "bat0": {
+                "interfaces": {
+                    "tunnel": [
+                        "de:ad:be:ff:05:05",
+                        "de:ad:be:fc:05:05",
+                        "de:ad:bf:ff:05:05"
+                    ]
+                }
+            }
+        }
+    }
+  },
+  {
+    "node_id": "00163efb9d8d",
+    "hostname": "gw03"
+  }
+]
+

generate_aliases.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python2
+
+from __future__ import print_function
+
+import json
+import os
+import sys
+
+if len(sys.argv) != 2:
+    print('usage: ' + sys.argv[0] + ' /path/to/peers')
+    sys.exit(1)
+
+peersDir = sys.argv[1]
+
+def normalizeMac(mac):
+    mac = mac.lower()
+    normalized = ''
+
+    n = 0
+
+    for c in mac:
+        if c != ':':
+            if n > 0 and n % 2 == 0:
+                normalized = normalized + ':'
+            normalized = normalized + c
+            n += 1
+
+    return normalized
+
+def toAlias(peer):
+    alias = {}
+
+    if not (peer.has_key('name') and peer.has_key('mac')):
+        return None
+
+    name = peer['name']
+    mac = peer['mac']
+
+    alias['node_id'] = mac.replace(':', '')
+    alias['hostname'] = name
+
+    if peer.has_key('geo'):
+        geo = peer['geo']
+
+        location = {}
+
+        if geo.has_key('lon'): location['longitude'] = geo['lon']
+        if geo.has_key('lat'): location['latitude'] = geo['lat']
+
+        alias['location'] = location
+
+    #alias['network'] = {}
+    #alias['network']['mesh_interfaces'] = [mac]
+
+    return alias
+
+aliases = []
+
+for filename in os.listdir(peersDir):
+    if len(filename) == 0 or filename[0] == '.':
+        continue
+
+    isGateway = False
+
+    absFilename = peersDir + '/' + filename
+    if os.path.isfile(absFilename):
+        peerFile = open(absFilename, 'r')
+        try:
+            peerLines = peerFile.readlines()
+            peer = {}
+
+            for line in peerLines:
+                parts = line.split()
+
+                if len(parts) > 2:
+                    if parts[1] == 'Knotenname:':
+                        peer['name'] = parts[2]
+
+                    elif parts[0] == 'remote':
+                        isGateway = True
+
+                    elif parts[1] == 'MAC:':
+                        peer['mac'] = normalizeMac(parts[2])
+
+                    elif parts[1] == 'Koordinaten:' and len(parts) > 3:
+                        try:
+                            peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
+
+                        except ValueError:
+                            print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
+
+                elif len(parts) == 2 and parts[0] == 'key':
+                    keyParts = parts[1].split('"')
+                    if len(keyParts) > 1:
+                        peer['vpn'] = keyParts[1].lower()
+
+            if isGateway:
+                continue
+
+            alias = toAlias(peer)
+            if alias:
+                aliases.append(alias)
+
+        except Exception as e:
+            print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
+
+        finally:
+            peerFile.close()
+
+print(json.dumps(aliases))

generate_aliases_v2.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python2
+
+from __future__ import print_function
+
+import json
+import os
+import sys
+
+if len(sys.argv) != 2:
+    print('usage: ' + sys.argv[0] + ' /path/to/peers')
+    sys.exit(1)
+
+peersDir = sys.argv[1]
+
+def normalizeMac(mac):
+    mac = mac.lower()
+    normalized = ''
+
+    n = 0
+
+    for c in mac:
+        if c != ':':
+            if n > 0 and n % 2 == 0:
+                normalized = normalized + ':'
+            normalized = normalized + c
+            n += 1
+
+    return normalized
+
+def toAlias(peer):
+    alias = {}
+
+    if not (peer.has_key('name') and peer.has_key('mac')):
+        return None
+
+    name = peer['name']
+    mac = peer['mac']
+
+    alias['node_id'] = mac.replace(':', '')
+    alias['hostname'] = name
+
+    if peer.has_key('geo'):
+        geo = peer['geo']
+
+        location = {}
+
+        if geo.has_key('lon'): location['longitude'] = geo['lon']
+        if geo.has_key('lat'): location['latitude'] = geo['lat']
+
+        alias['location'] = location
+
+    #alias['network'] = {}
+    #alias['network']['mesh_interfaces'] = [mac]
+
+    return {'nodeinfo':alias}
+
+aliases = {}
+
+for filename in os.listdir(peersDir):
+    if len(filename) == 0 or filename[0] == '.':
+        continue
+
+    isGateway = False
+
+    absFilename = peersDir + '/' + filename
+    if os.path.isfile(absFilename):
+        peerFile = open(absFilename, 'r')
+        try:
+            peerLines = peerFile.readlines()
+            peer = {}
+
+            for line in peerLines:
+                parts = line.split()
+
+                if len(parts) > 2:
+                    if parts[1] == 'Knotenname:':
+                        peer['name'] = parts[2]
+
+                    elif parts[0] == 'remote':
+                        isGateway = True
+
+                    elif parts[1] == 'MAC:':
+                        peer['mac'] = normalizeMac(parts[2])
+
+                    elif parts[1] == 'Koordinaten:' and len(parts) > 3:
+                        try:
+                            peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
+
+                        except ValueError:
+                            print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
+
+                elif len(parts) == 2 and parts[0] == 'key':
+                    keyParts = parts[1].split('"')
+                    if len(keyParts) > 1:
+                        peer['vpn'] = keyParts[1].lower()
+
+            if isGateway:
+                continue
+
+            alias = toAlias(peer)
+            if alias:
+                tmpid = alias['nodeinfo']['node_id']
+#                alias['nodeinfo'].pop('node_id')
+                aliases[tmpid] = alias
+
+        except Exception as e:
+            print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
+
+        finally:
+            peerFile.close()
+
+print(json.dumps(aliases))

lib/alfred.py

@@ -13,7 +13,7 @@ class Alfred(object):
             raise RuntimeError('alfred: invalid unix socket path given')
 
     def _fetch(self, data_type):
-        cmd = ['alfred-json',
+        cmd = ['/usr/local/bin/alfred-json',
                '-z',
                '-f', 'json',
                '-r', str(data_type)]

lib/batman.py

@@ -1,5 +1,6 @@
 import subprocess
 import json
+import os
 import re
 
 
@@ -12,6 +13,14 @@ class Batman(object):
         self.mesh_interface = mesh_interface
         self.alfred_sock = alfred_sockpath
 
+        # ensure /usr/sbin and /usr/local/sbin are in PATH
+        env = os.environ
+        path = set(env['PATH'].split(':'))
+        path.add('/usr/sbin/')
+        path.add('/usr/local/sbin')
+        env['PATH'] = ':'.join(path)
+        self.environ = env
+
         # compile regular expressions only once on startup
         self.mac_addr_pattern = re.compile(r'(([a-z0-9]{2}:){5}[a-z0-9]{2})')
 
@@ -37,7 +46,7 @@ class Batman(object):
         cmd = ['batadv-vis', '-i', self.mesh_interface, '-f', 'json']
         if self.alfred_sock:
             cmd.extend(['-u', self.alfred_sock])
-        output = subprocess.check_output(cmd)
+        output = subprocess.check_output(cmd, env=self.environ)
         lines = output.splitlines()
         return self.vis_data_helper(lines)
 
@@ -46,8 +55,10 @@ class Batman(object):
         Parse "batctl -m <mesh_interface> gwl -n"
         into an array of dictionaries.
         """
-        output = subprocess.check_output(
+        cmd = ['batctl', '-m', self.mesh_interface, 'gwl', '-n']
-            ['batctl', '-m', self.mesh_interface, 'gwl', '-n'])
+        if os.geteuid() > 0:
+            cmd.insert(0, 'sudo')
+        output = subprocess.check_output(cmd, env=self.environ)
         output_utf8 = output.decode('utf-8')
         rows = output_utf8.splitlines()
 
@@ -73,8 +84,10 @@ class Batman(object):
         Parse "batctl -m <mesh_interface> gw"
         return: tuple mode, bandwidth, if mode != server then bandwidth is None
         """
-        output = subprocess.check_output(
+        cmd = ['batctl', '-m', self.mesh_interface, 'gw']
-            ['batctl', '-m', self.mesh_interface, 'gw'])
+        if os.geteuid() > 0:
+            cmd.insert(0, 'sudo')
+        output = subprocess.check_output(cmd, env=self.environ)
         chunks = output.decode("utf-8").split()
 
         return chunks[0], chunks[3] if 3 in chunks else None
@@ -83,7 +96,3 @@ if __name__ == "__main__":
     bc = Batman()
     vd = bc.vis_data()
     gw = bc.gateway_list()
-    for x in vd:
-        print(x)
-    print(gw)
-    print(bc.gateway_mode())

lib/graph.py

@@ -25,7 +25,6 @@ def mark_vpn(graph, vpn_macs):
     components = map(frozenset, nx.weakly_connected_components(graph))
     components = filter(vpn_macs.intersection, components)
     nodes = reduce(lambda a, b: a | b, components, set())
-
     for node in nodes:
         for k, v in graph[node].items():
             v['vpn'] = True

lib/nodelist.py

@@ -0,0 +1,27 @@
+def export_nodelist(now, nodedb):
+    nodelist = list()
+
+    for node_id, node in nodedb["nodes"].items():
+        node_out = dict()
+        node_out["id"] = node_id
+        node_out["name"] = node["nodeinfo"]["hostname"]
+
+        if "location" in node["nodeinfo"]:
+            node_out["position"] = {"lat": node["nodeinfo"]["location"]["latitude"],
+                                    "long": node["nodeinfo"]["location"]["longitude"]}
+
+        node_out["status"] = dict()
+        node_out["status"]["online"] = node["flags"]["online"]
+
+        if "firstseen" in node:
+            node_out["status"]["firstcontact"] = node["firstseen"]
+
+        if "lastseen" in node:
+            node_out["status"]["lastcontact"] = node["lastseen"]
+
+        if "clients" in node["statistics"]:
+            node_out["status"]["clients"] = node["statistics"]["clients"]
+
+        nodelist.append(node_out)
+
+    return {"version": "1.0.1", "nodes": nodelist, "updated_at": now.isoformat()}

lib/nodes.py

@@ -6,11 +6,39 @@ from functools import reduce
 def build_mac_table(nodes):
     macs = dict()
     for node_id, node in nodes.items():
+        try:
+            macs[node['network']['mac']] = node_id
+        except KeyError:
+            pass
         try:
             for mac in node['nodeinfo']['network']['mesh_interfaces']:
                 macs[mac] = node_id
         except KeyError:
             pass
+
+        try:
+            for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['wireless']:
+                macs[mac] = node_id
+        except KeyError:
+            pass
+
+        try:
+            for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['tunnel']:
+                macs[mac] = node_id
+        except KeyError:
+            pass
+        try:
+            for mac in node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']:
+                macs[mac] = node_id
+        except KeyError:
+            pass
+
+        try:
+            for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']:
+                macs[mac] = node_id
+        except KeyError:
+            pass
+
     return macs
 
 
@@ -37,12 +65,23 @@ def mark_online(node, now):
     node['flags']['online'] = True
 
 
-def import_nodeinfo(nodes, nodeinfos, now, assume_online=False):
+def overrideFields(dest, src, fields):
+    for field in fields:
+        if field in src:
+            dest[field] = src[field]
+        else:
+            dest.pop(field, None)
+
+
+def import_nodeinfo(nodes, nodeinfos, now, assume_online=False, statics=False):
     for nodeinfo in filter(lambda d: 'node_id' in d, nodeinfos):
-        node = nodes.setdefault(nodeinfo['node_id'], {'flags': dict()})
+        node = nodes.setdefault(nodeinfo['node_id'], {'flags': {'online': False, 'gateway': False}})
-        node['nodeinfo'] = nodeinfo
+
-        node['flags']['online'] = False
+        if statics:
-        node['flags']['gateway'] = False
+            node['nodeinfo'] = node.setdefault('nodeinfo', {})
+            overrideFields(node['nodeinfo'], nodeinfo, ['hostname', 'location', 'node_id'])
+        else:
+            node['nodeinfo'] = nodeinfo
 
         if assume_online:
             mark_online(node, now)
@@ -59,7 +98,7 @@ def import_statistics(nodes, stats):
             node['statistics'][target] = f(reduce(dict.__getitem__,
                                                   source,
                                                   statistics))
-        except (KeyError, TypeError):
+        except (KeyError, TypeError, ZeroDivisionError):
             pass
 
     macs = build_mac_table(nodes)
@@ -73,6 +112,7 @@ def import_statistics(nodes, stats):
         add(node, stats, 'memory_usage', ['memory'],
             lambda d: 1 - d['free'] / d['total'])
         add(node, stats, 'rootfs_usage', ['rootfs_usage'])
+        add(node, stats, 'traffic', ['traffic'])
 
 
 def import_mesh_ifs_vis_data(nodes, vis_data):
@@ -97,12 +137,34 @@ def import_mesh_ifs_vis_data(nodes, vis_data):
     for v in mesh_nodes:
         node = v[0]
 
-        try:
+        ifs = set()
-            mesh_ifs = set(node['nodeinfo']['network']['mesh_interfaces'])
-        except KeyError:
-            mesh_ifs = set()
 
-        node['nodeinfo']['network']['mesh_interfaces'] = list(mesh_ifs | v[1])
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh_interfaces']))
+        except KeyError:
+            pass
+
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['wireless']))
+        except KeyError:
+            pass
+
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['tunnel']))
+        except KeyError:
+            pass
+
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']))
+        except KeyError:
+            pass
+
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']))
+        except KeyError:
+            pass
+
+        node['nodeinfo']['network']['mesh_interfaces'] = list(ifs | v[1])
 
 
 def import_vis_clientcount(nodes, vis_data):
@@ -118,7 +180,6 @@ def import_vis_clientcount(nodes, vis_data):
 def mark_gateways(nodes, gateways):
     macs = build_mac_table(nodes)
     gateways = filter(lambda d: d in macs, gateways)
-
     for node in map(lambda d: nodes[macs[d]], gateways):
         node['flags']['gateway'] = True
 

lib/validate.py

@@ -0,0 +1,19 @@
+import json
+
+
+def validate_nodeinfos(nodeinfos):
+    result = []
+
+    for nodeinfo in nodeinfos:
+        if validate_nodeinfo(nodeinfo):
+            result.append(nodeinfo)
+
+    return result
+
+
+def validate_nodeinfo(nodeinfo):
+    if 'location' in nodeinfo:
+        if 'latitude' not in nodeinfo['location'] or 'longitude' not in nodeinfo['location']:
+            return False
+
+    return True

mkmap.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+FFMAPPATH='/opt/ffmap-backend/'
+PEERS="/etc/fastd/ffhh-mesh-vpn/peers"
+
+python2 $FFMAPPATH/generate_aliases.py $PEERS > $FFMAPPATH/aliases.json
+#python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 --vpn de:ad:be:ff:05:05 --vpn de:ad:be:ff:05:06 --vpn de:ad:be:ff:03:03 --vpn de:ad:be:ff:22:22 --vpn de:ad:be:ff:22:23 --vpn de:ad:be:ff:88:88 --vpn de:ad:be:ff:88:89 --vpn de:ad:bf:ff:88:88 --vpn de:ad:bf:ff:22:22 --vpn de:ad:bf:ff:03:03 --vpn de:ad:bf:ff:05:05 --vpn de:ad:bf:ff:01:01 --vpn de:ad:be:fc:03:03 --vpn 00:16:3e:53:75:0d --vpn de:ad:be:fc:05:05 --vpn de:ad:be:fc:01:01 --vpn de:ad:be:ef:03:03 --vpn de:ad:be:ef:01:01 --vpn de:ad:be:ef:05:05 --vpn 00:16:3e:fb:9d:8d --vpn 00:16:3e:fb:9d:9d
+python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 de:ad:be:ff:05:05 de:ad:be:ff:05:06 de:ad:be:ff:03:03 de:ad:be:ff:22:22 de:ad:be:ff:22:23 de:ad:be:ff:88:88 de:ad:be:ff:88:89 de:ad:bf:ff:88:88 de:ad:bf:ff:22:22 de:ad:bf:ff:03:03 de:ad:bf:ff:05:05 de:ad:bf:ff:01:01 de:ad:be:fc:03:03 00:16:3e:53:75:0d de:ad:be:fc:05:05 de:ad:be:fc:01:01 de:ad:be:ef:03:03 de:ad:be:ef:01:01 de:ad:be:ef:05:05 00:16:3e:fb:9d:8d 00:16:3e:fb:9d:9d

node_number.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#Bibliotheken importieren
+import time
+import datetime
+import json
+import urllib2
+
+#Datei oeffnen
+Datei = urllib2.urlopen('https://map.hamburg.freifunk.net/nodes.json')
+Datei_Sued = urllib2.urlopen('https://map.hamburg.freifunk.net/hhsued/mv1/nodes.json')
+
+Text = Datei.read()
+Knotenzahl = Text.count('"online": true')
+Text = Datei_Sued.read()
+Knotenzahl = Knotenzahl + Text.count('"online":true')
+
+#Zeit holen
+thetime = datetime.datetime.now().isoformat()
+
+ffhh = None
+
+#Freifunk API-Datei einladen und JSON lesen
+with open('/var/www/meta/ffhh.json', 'r') as fp:
+        ffhh = json.load(fp)
+
+#Attribute Zeitstempel und Knotenanzahl setzen
+ffhh['state']['lastchange'] = thetime
+ffhh['state']['nodes'] = Knotenzahl
+
+#Freifunk API-Datein mit geaenderten werten schreiben
+with open('/var/www/meta/ffhh.json', 'w') as fp:
+        json.dump(ffhh, fp, indent=2, separators=(',', ': '))