Support for "bat-ffhh" interface, add generate_aliases_v2.py

- Von nodelist.json auf nodes.json gewechselt
- nodes.json für Süd Domäne hinzugefügt
- Liest nodes.json's nun aus URLs, da sie auf verschiedenen servern liegen
- Durchsucht die nodes.json's nur noch als Strings, statt JSON auszuwerten

alfred_merge.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import subprocess
+import json
+
+from collections import MutableMapping
+
+def rec_merge(d1, d2):
+    '''
+    Update two dicts of dicts recursively, 
+    if either mapping has leaves that are non-dicts, 
+    the second's leaf overwrites the first's.
+    '''
+    for k, v in d1.items(): # in Python 2, use .iteritems()!
+        if k in d2:
+            # this next check is the only difference!
+            if all(isinstance(e, MutableMapping) for e in (v, d2[k])):
+                d2[k] = rec_merge(v, d2[k])
+            # we could further check types and merge as appropriate here.
+    d3 = d1.copy()
+    d3.update(d2)
+    return d3
+
+
+class alfred_merge:
+  def __init__(self,request_data_type_1 = 158, request_data_type_2 = 159):
+    self.request_data_type_1 = request_data_type_1
+    self.request_data_type_2 = request_data_type_2
+
+  def aliases(self):
+    output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_1),"-f","json"])
+    alfred_data_1 = json.loads(output.decode("utf-8"))
+    output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_2),"-f","json"])
+    alfred_data_2 = json.loads(output.decode("utf-8"))
+    
+    return json.dumps(rec_merge(alfred_data_1, alfred_data_2))
+
+
+if __name__ == "__main__":
+  ad = alfred_merge()
+  al = ad.aliases()
+  print(al)
+

backend.py

@@ -27,8 +27,13 @@ def main(params):
     os.makedirs(params['dest_dir'], exist_ok=True)
 
     nodes_fn = os.path.join(params['dest_dir'], 'nodes.json')
+    tmp_nodes_fn = os.path.join(params['dest_dir'], 'nodes.json.tmp')
+
     graph_fn = os.path.join(params['dest_dir'], 'graph.json')
+    tmp_graph_fn = os.path.join(params['dest_dir'], 'graph.json.tmp')
+
     nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json')
+    tmp_nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json.tmp')
 
     now = datetime.utcnow().replace(microsecond=0)
 
@@ -84,9 +89,9 @@ def main(params):
     # integrate static aliases data
     for aliases in params['aliases']:
         with open(aliases, 'r') as f:
-            nodeinfo = validate_nodeinfos(json.load(f))
+#           nodeinfo = validate_nodeinfos(json.load(f))
-            nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
+            nodes.import_nodeinfo(nodedb['nodes'], json.load(f),
-                                  now, assume_online=False)
+                                  now, assume_online=False, statics=True)
 
     nodes.reset_statistics(nodedb['nodes'])
     for alfred in alfred_instances:
@@ -97,7 +102,6 @@ def main(params):
     for batman in batman_instances:
         vd = batman.vis_data()
         gwl = batman.gateway_list()
-
         mesh_info.append((vd, gwl))
 
     # update nodedb from batman-adv data
@@ -126,6 +130,8 @@ def main(params):
             try:
                 for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
                     macs.add(mac)
+                for mac in node["nodeinfo"]["network"]["mesh"]["bat-ffhh"]["interfaces"]["tunnel"]:
+                    macs.add(mac)
             except KeyError:
                 pass
 
@@ -137,27 +143,30 @@ def main(params):
     batadv_graph = graph.to_undirected(batadv_graph)
 
     # write processed data to dest dir
-    with open(nodes_fn, 'w') as f:
+    with open(tmp_nodes_fn, 'w') as f:
         json.dump(nodedb, f)
 
     graph_out = {'batadv': json_graph.node_link_data(batadv_graph),
                  'version': GRAPH_VERSION}
 
-    with open(graph_fn, 'w') as f:
+    with open(tmp_graph_fn, 'w') as f:
         json.dump(graph_out, f)
 
-    with open(nodelist_fn, 'w') as f:
+    with open(tmp_nodelist_fn, 'w') as f:
         json.dump(export_nodelist(now, nodedb), f)
 
+    os.rename(tmp_nodes_fn, nodes_fn)
+    os.rename(tmp_graph_fn, graph_fn)
+    os.rename(tmp_nodelist_fn, nodelist_fn)
+
     # optional rrd graphs (trigger with --rrd)
     if params['rrd']:
         script_directory = os.path.dirname(os.path.realpath(__file__))
         rrd = RRD(os.path.join(script_directory, 'nodedb'),
                   os.path.join(params['dest_dir'], 'nodes'))
-        rrd.update_database(nodedb['nodes'], batadv_graph)
+        rrd.update_database(nodedb['nodes'])
         rrd.update_images()
 
-
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
 

gateway.json

@@ -0,0 +1,29 @@
+[
+  {
+    "node_id": "deadbfff0101",
+    "hostname": "gw01"
+  },
+  {
+    "node_id": "deadbeef0505",
+    "hostname": "gw02.hamburg.freifunk.net",
+    "network": {
+        "mac": "de:ad:be:ef:05:05",
+        "mesh": {
+            "bat0": {
+                "interfaces": {
+                    "tunnel": [
+                        "de:ad:be:ff:05:05",
+                        "de:ad:be:fc:05:05",
+                        "de:ad:bf:ff:05:05"
+                    ]
+                }
+            }
+        }
+    }
+  },
+  {
+    "node_id": "00163efb9d8d",
+    "hostname": "gw03"
+  }
+]
+

generate_aliases.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python2
+
+from __future__ import print_function
+
+import json
+import os
+import sys
+
+if len(sys.argv) != 2:
+    print('usage: ' + sys.argv[0] + ' /path/to/peers')
+    sys.exit(1)
+
+peersDir = sys.argv[1]
+
+def normalizeMac(mac):
+    mac = mac.lower()
+    normalized = ''
+
+    n = 0
+
+    for c in mac:
+        if c != ':':
+            if n > 0 and n % 2 == 0:
+                normalized = normalized + ':'
+            normalized = normalized + c
+            n += 1
+
+    return normalized
+
+def toAlias(peer):
+    alias = {}
+
+    if not (peer.has_key('name') and peer.has_key('mac')):
+        return None
+
+    name = peer['name']
+    mac = peer['mac']
+
+    alias['node_id'] = mac.replace(':', '')
+    alias['hostname'] = name
+
+    if peer.has_key('geo'):
+        geo = peer['geo']
+
+        location = {}
+
+        if geo.has_key('lon'): location['longitude'] = geo['lon']
+        if geo.has_key('lat'): location['latitude'] = geo['lat']
+
+        alias['location'] = location
+
+    #alias['network'] = {}
+    #alias['network']['mesh_interfaces'] = [mac]
+
+    return alias
+
+aliases = []
+
+for filename in os.listdir(peersDir):
+    if len(filename) == 0 or filename[0] == '.':
+        continue
+
+    isGateway = False
+
+    absFilename = peersDir + '/' + filename
+    if os.path.isfile(absFilename):
+        peerFile = open(absFilename, 'r')
+        try:
+            peerLines = peerFile.readlines()
+            peer = {}
+
+            for line in peerLines:
+                parts = line.split()
+
+                if len(parts) > 2:
+                    if parts[1] == 'Knotenname:':
+                        peer['name'] = parts[2]
+
+                    elif parts[0] == 'remote':
+                        isGateway = True
+
+                    elif parts[1] == 'MAC:':
+                        peer['mac'] = normalizeMac(parts[2])
+
+                    elif parts[1] == 'Koordinaten:' and len(parts) > 3:
+                        try:
+                            peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
+
+                        except ValueError:
+                            print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
+
+                elif len(parts) == 2 and parts[0] == 'key':
+                    keyParts = parts[1].split('"')
+                    if len(keyParts) > 1:
+                        peer['vpn'] = keyParts[1].lower()
+
+            if isGateway:
+                continue
+
+            alias = toAlias(peer)
+            if alias:
+                aliases.append(alias)
+
+        except Exception as e:
+            print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
+
+        finally:
+            peerFile.close()
+
+print(json.dumps(aliases))

generate_aliases_v2.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python2
+
+from __future__ import print_function
+
+import json
+import os
+import sys
+
+if len(sys.argv) != 2:
+    print('usage: ' + sys.argv[0] + ' /path/to/peers')
+    sys.exit(1)
+
+peersDir = sys.argv[1]
+
+def normalizeMac(mac):
+    mac = mac.lower()
+    normalized = ''
+
+    n = 0
+
+    for c in mac:
+        if c != ':':
+            if n > 0 and n % 2 == 0:
+                normalized = normalized + ':'
+            normalized = normalized + c
+            n += 1
+
+    return normalized
+
+def toAlias(peer):
+    alias = {}
+
+    if not (peer.has_key('name') and peer.has_key('mac')):
+        return None
+
+    name = peer['name']
+    mac = peer['mac']
+
+    alias['node_id'] = mac.replace(':', '')
+    alias['hostname'] = name
+
+    if peer.has_key('geo'):
+        geo = peer['geo']
+
+        location = {}
+
+        if geo.has_key('lon'): location['longitude'] = geo['lon']
+        if geo.has_key('lat'): location['latitude'] = geo['lat']
+
+        alias['location'] = location
+
+    #alias['network'] = {}
+    #alias['network']['mesh_interfaces'] = [mac]
+
+    return {'nodeinfo':alias}
+
+aliases = {}
+
+for filename in os.listdir(peersDir):
+    if len(filename) == 0 or filename[0] == '.':
+        continue
+
+    isGateway = False
+
+    absFilename = peersDir + '/' + filename
+    if os.path.isfile(absFilename):
+        peerFile = open(absFilename, 'r')
+        try:
+            peerLines = peerFile.readlines()
+            peer = {}
+
+            for line in peerLines:
+                parts = line.split()
+
+                if len(parts) > 2:
+                    if parts[1] == 'Knotenname:':
+                        peer['name'] = parts[2]
+
+                    elif parts[0] == 'remote':
+                        isGateway = True
+
+                    elif parts[1] == 'MAC:':
+                        peer['mac'] = normalizeMac(parts[2])
+
+                    elif parts[1] == 'Koordinaten:' and len(parts) > 3:
+                        try:
+                            peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
+
+                        except ValueError:
+                            print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
+
+                elif len(parts) == 2 and parts[0] == 'key':
+                    keyParts = parts[1].split('"')
+                    if len(keyParts) > 1:
+                        peer['vpn'] = keyParts[1].lower()
+
+            if isGateway:
+                continue
+
+            alias = toAlias(peer)
+            if alias:
+                tmpid = alias['nodeinfo']['node_id']
+#                alias['nodeinfo'].pop('node_id')
+                aliases[tmpid] = alias
+
+        except Exception as e:
+            print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
+
+        finally:
+            peerFile.close()
+
+print(json.dumps(aliases))

lib/NodeRRD.py

@@ -8,35 +8,25 @@ class NodeRRD(RRD):
     ds_list = [
         DS('upstate', 'GAUGE', 120, 0, 1),
         DS('clients', 'GAUGE', 120, 0, float('NaN')),
-        DS('neighbors', 'GAUGE', 120, 0, float('NaN')),
-        DS('vpn_neighbors', 'GAUGE', 120, 0, float('NaN')),
-        DS('loadavg', 'GAUGE', 120, 0, float('NaN')),
-        DS('rx_bytes', 'DERIVE', 120, 0, float('NaN')),
-        DS('rx_packets', 'DERIVE', 120, 0, float('NaN')),
-        DS('tx_bytes', 'DERIVE', 120, 0, float('NaN')),
-        DS('tx_packets', 'DERIVE', 120, 0, float('NaN')),
-        DS('mgmt_rx_bytes', 'DERIVE', 120, 0, float('NaN')),
-        DS('mgmt_rx_packets', 'DERIVE', 120, 0, float('NaN')),
-        DS('mgmt_tx_bytes', 'DERIVE', 120, 0, float('NaN')),
-        DS('mgmt_tx_packets', 'DERIVE', 120, 0, float('NaN')),
-        DS('forward_bytes', 'DERIVE', 120, 0, float('NaN')),
-        DS('forward_packets', 'DERIVE', 120, 0, float('NaN')),
     ]
     rra_list = [
         # 2 hours of  1 minute samples
         RRA('AVERAGE', 0.5, 1, 120),
-        # 7 days  of  15 minute samples
+        #  5 days  of  5 minute samples
-        RRA('AVERAGE', 0.5, 15, 672),
+        RRA('AVERAGE', 0.5, 5, 1440),
+        # 30 days  of  1 hour   samples
+        RRA('AVERAGE', 0.5, 60, 720),
+        #  1 year  of 12 hour   samples
+        RRA('AVERAGE', 0.5, 720, 730),
     ]
 
-    def __init__(self, filename, node=None, graph=None):
+    def __init__(self, filename, node=None):
         """
         Create a new RRD for a given node.
 
         If the RRD isn't supposed to be updated, the node can be omitted.
         """
         self.node = node
-        self.node_graph = graph
         super().__init__(filename)
         self.ensure_sanity(self.ds_list, self.rra_list, step=60)
 
@@ -47,28 +37,8 @@ class NodeRRD(RRD):
 
     # TODO: fix this, python does not support function overloading
     def update(self):
-        values = {
+        super().update({'upstate': int(self.node['flags']['online']),
-            'upstate': int(self.node['flags']['online']),
+                        'clients': self.node['statistics']['clients']})
-            'clients': float(self.node['statistics']['clients']),
-            'loadavg': float(self.node['statistics'].get('loadavg', 0)),
-        }
-        for item in ('rx', 'tx', 'mgmt_rx', 'mgmt_tx', 'forward'):
-            try:
-                values.update({
-                    ('%s_bytes' % item): int(self.node['statistics'].get('traffic', {}).get(item, {}).get('bytes', 0)),
-                    ('%s_packets' % item): int(self.node['statistics'].get('traffic', {}).get(item, {}).get('packets', 0)),
-                })
-            except TypeError:
-                pass
-        try:
-            graph_node = next(key for key, node in self.node_graph.nodes(data=True) if node.get('node_id') == self.node['nodeinfo']['node_id'])
-            values.update({
-                'neighbors': float(len(self.node_graph[graph_node])),
-                'vpn_neighbors': float(len(list(filter(lambda edge: edge.get('vpn', False), self.node_graph[graph_node].values())))),
-            })
-        except StopIteration:
-            pass
-        super().update(values)
 
     def graph(self, directory, timeframe):
         """

lib/alfred.py

@@ -13,7 +13,7 @@ class Alfred(object):
             raise RuntimeError('alfred: invalid unix socket path given')
 
     def _fetch(self, data_type):
-        cmd = ['alfred-json',
+        cmd = ['/usr/local/bin/alfred-json',
                '-z',
                '-f', 'json',
                '-r', str(data_type)]

lib/batman.py

@@ -96,7 +96,3 @@ if __name__ == "__main__":
     bc = Batman()
     vd = bc.vis_data()
     gw = bc.gateway_list()
-    for x in vd:
-        print(x)
-    print(gw)
-    print(bc.gateway_mode())

lib/graph.py

@@ -25,7 +25,6 @@ def mark_vpn(graph, vpn_macs):
     components = map(frozenset, nx.weakly_connected_components(graph))
     components = filter(vpn_macs.intersection, components)
     nodes = reduce(lambda a, b: a | b, components, set())
-
     for node in nodes:
         for k, v in graph[node].items():
             v['vpn'] = True

lib/nodelist.py

@@ -13,6 +13,9 @@ def export_nodelist(now, nodedb):
         node_out["status"] = dict()
         node_out["status"]["online"] = node["flags"]["online"]
 
+        if "firstseen" in node:
+            node_out["status"]["firstcontact"] = node["firstseen"]
+
         if "lastseen" in node:
             node_out["status"]["lastcontact"] = node["lastseen"]
 

lib/nodes.py

@@ -6,6 +6,10 @@ from functools import reduce
 def build_mac_table(nodes):
     macs = dict()
     for node_id, node in nodes.items():
+        try:
+            macs[node['network']['mac']] = node_id
+        except KeyError:
+            pass
         try:
             for mac in node['nodeinfo']['network']['mesh_interfaces']:
                 macs[mac] = node_id
@@ -23,6 +27,11 @@ def build_mac_table(nodes):
                 macs[mac] = node_id
         except KeyError:
             pass
+        try:
+            for mac in node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']:
+                macs[mac] = node_id
+        except KeyError:
+            pass
 
         try:
             for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']:
@@ -56,12 +65,23 @@ def mark_online(node, now):
     node['flags']['online'] = True
 
 
-def import_nodeinfo(nodes, nodeinfos, now, assume_online=False):
+def overrideFields(dest, src, fields):
+    for field in fields:
+        if field in src:
+            dest[field] = src[field]
+        else:
+            dest.pop(field, None)
+
+
+def import_nodeinfo(nodes, nodeinfos, now, assume_online=False, statics=False):
     for nodeinfo in filter(lambda d: 'node_id' in d, nodeinfos):
-        node = nodes.setdefault(nodeinfo['node_id'], {'flags': dict()})
+        node = nodes.setdefault(nodeinfo['node_id'], {'flags': {'online': False, 'gateway': False}})
-        node['nodeinfo'] = nodeinfo
+
-        node['flags']['online'] = False
+        if statics:
-        node['flags']['gateway'] = False
+            node['nodeinfo'] = node.setdefault('nodeinfo', {})
+            overrideFields(node['nodeinfo'], nodeinfo, ['hostname', 'location', 'node_id'])
+        else:
+            node['nodeinfo'] = nodeinfo
 
         if assume_online:
             mark_online(node, now)
@@ -134,6 +154,11 @@ def import_mesh_ifs_vis_data(nodes, vis_data):
         except KeyError:
             pass
 
+        try:
+            ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']))
+        except KeyError:
+            pass
+
         try:
             ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']))
         except KeyError:
@@ -155,7 +180,6 @@ def import_vis_clientcount(nodes, vis_data):
 def mark_gateways(nodes, gateways):
     macs = build_mac_table(nodes)
     gateways = filter(lambda d: d in macs, gateways)
-
     for node in map(lambda d: nodes[macs[d]], gateways):
         node['flags']['gateway'] = True
 

lib/rrddb.py

@@ -27,7 +27,7 @@ class RRD(object):
         except OSError:
             os.mkdir(self.imagePath)
 
-    def update_database(self, nodes, graph):
+    def update_database(self, nodes):
         online_nodes = dict(filter(
             lambda d: d[1]['flags']['online'], nodes.items()))
         client_count = sum(map(
@@ -35,7 +35,7 @@ class RRD(object):
 
         self.globalDb.update(len(online_nodes), client_count)
         for node_id, node in online_nodes.items():
-            rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node, graph)
+            rrd = NodeRRD(os.path.join(self.dbPath, node_id + '.rrd'), node)
             rrd.update()
 
     def update_images(self):

mkmap.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+FFMAPPATH='/opt/ffmap-backend/'
+PEERS="/etc/fastd/ffhh-mesh-vpn/peers"
+
+python2 $FFMAPPATH/generate_aliases.py $PEERS > $FFMAPPATH/aliases.json
+#python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 --vpn de:ad:be:ff:05:05 --vpn de:ad:be:ff:05:06 --vpn de:ad:be:ff:03:03 --vpn de:ad:be:ff:22:22 --vpn de:ad:be:ff:22:23 --vpn de:ad:be:ff:88:88 --vpn de:ad:be:ff:88:89 --vpn de:ad:bf:ff:88:88 --vpn de:ad:bf:ff:22:22 --vpn de:ad:bf:ff:03:03 --vpn de:ad:bf:ff:05:05 --vpn de:ad:bf:ff:01:01 --vpn de:ad:be:fc:03:03 --vpn 00:16:3e:53:75:0d --vpn de:ad:be:fc:05:05 --vpn de:ad:be:fc:01:01 --vpn de:ad:be:ef:03:03 --vpn de:ad:be:ef:01:01 --vpn de:ad:be:ef:05:05 --vpn 00:16:3e:fb:9d:8d --vpn 00:16:3e:fb:9d:9d
+python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 de:ad:be:ff:05:05 de:ad:be:ff:05:06 de:ad:be:ff:03:03 de:ad:be:ff:22:22 de:ad:be:ff:22:23 de:ad:be:ff:88:88 de:ad:be:ff:88:89 de:ad:bf:ff:88:88 de:ad:bf:ff:22:22 de:ad:bf:ff:03:03 de:ad:bf:ff:05:05 de:ad:bf:ff:01:01 de:ad:be:fc:03:03 00:16:3e:53:75:0d de:ad:be:fc:05:05 de:ad:be:fc:01:01 de:ad:be:ef:03:03 de:ad:be:ef:01:01 de:ad:be:ef:05:05 00:16:3e:fb:9d:8d 00:16:3e:fb:9d:9d

node_number.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#Bibliotheken importieren
+import time
+import datetime
+import json
+import urllib2
+
+#Datei oeffnen
+Datei = urllib2.urlopen('https://map.hamburg.freifunk.net/nodes.json')
+Datei_Sued = urllib2.urlopen('https://map.hamburg.freifunk.net/hhsued/mv1/nodes.json')
+
+Text = Datei.read()
+Knotenzahl = Text.count('"online": true')
+Text = Datei_Sued.read()
+Knotenzahl = Knotenzahl + Text.count('"online":true')
+
+#Zeit holen
+thetime = datetime.datetime.now().isoformat()
+
+ffhh = None
+
+#Freifunk API-Datei einladen und JSON lesen
+with open('/var/www/meta/ffhh.json', 'r') as fp:
+        ffhh = json.load(fp)
+
+#Attribute Zeitstempel und Knotenanzahl setzen
+ffhh['state']['lastchange'] = thetime
+ffhh['state']['nodes'] = Knotenzahl
+
+#Freifunk API-Datein mit geaenderten werten schreiben
+with open('/var/www/meta/ffhh.json', 'w') as fp:
+        json.dump(ffhh, fp, indent=2, separators=(',', ': '))