diff --git a/alfred_merge.py b/alfred_merge.py deleted file mode 100755 index ee1143f..0000000 --- a/alfred_merge.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -import subprocess -import json - -from collections import MutableMapping - -def rec_merge(d1, d2): - ''' - Update two dicts of dicts recursively, - if either mapping has leaves that are non-dicts, - the second's leaf overwrites the first's. - ''' - for k, v in d1.items(): # in Python 2, use .iteritems()! - if k in d2: - # this next check is the only difference! - if all(isinstance(e, MutableMapping) for e in (v, d2[k])): - d2[k] = rec_merge(v, d2[k]) - # we could further check types and merge as appropriate here. - d3 = d1.copy() - d3.update(d2) - return d3 - - -class alfred_merge: - def __init__(self,request_data_type_1 = 158, request_data_type_2 = 159): - self.request_data_type_1 = request_data_type_1 - self.request_data_type_2 = request_data_type_2 - - def aliases(self): - output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_1),"-f","json"]) - alfred_data_1 = json.loads(output.decode("utf-8")) - output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_2),"-f","json"]) - alfred_data_2 = json.loads(output.decode("utf-8")) - - return json.dumps(rec_merge(alfred_data_1, alfred_data_2)) - - -if __name__ == "__main__": - ad = alfred_merge() - al = ad.aliases() - print(al) - diff --git a/backend.py b/backend.py index 8b9d9a6..3ea2f5d 100755 --- a/backend.py +++ b/backend.py @@ -27,13 +27,8 @@ def main(params): os.makedirs(params['dest_dir'], exist_ok=True) nodes_fn = os.path.join(params['dest_dir'], 'nodes.json') - tmp_nodes_fn = os.path.join(params['dest_dir'], 'nodes.json.tmp') - graph_fn = os.path.join(params['dest_dir'], 'graph.json') - tmp_graph_fn = os.path.join(params['dest_dir'], 'graph.json.tmp') - nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json') - tmp_nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json.tmp') now = datetime.utcnow().replace(microsecond=0) @@ -89,9 +84,9 @@ def main(params): # integrate static aliases data for aliases in params['aliases']: with open(aliases, 'r') as f: -# nodeinfo = validate_nodeinfos(json.load(f)) - nodes.import_nodeinfo(nodedb['nodes'], json.load(f), - now, assume_online=False, statics=True) + nodeinfo = validate_nodeinfos(json.load(f)) + nodes.import_nodeinfo(nodedb['nodes'], nodeinfo, + now, assume_online=False) nodes.reset_statistics(nodedb['nodes']) for alfred in alfred_instances: @@ -102,6 +97,7 @@ def main(params): for batman in batman_instances: vd = batman.vis_data() gwl = batman.gateway_list() + mesh_info.append((vd, gwl)) # update nodedb from batman-adv data @@ -130,8 +126,6 @@ def main(params): try: for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]: macs.add(mac) - for mac in node["nodeinfo"]["network"]["mesh"]["bat-ffhh"]["interfaces"]["tunnel"]: - macs.add(mac) except KeyError: pass @@ -143,22 +137,18 @@ def main(params): batadv_graph = graph.to_undirected(batadv_graph) # write processed data to dest dir - with open(tmp_nodes_fn, 'w') as f: + with open(nodes_fn, 'w') as f: json.dump(nodedb, f) graph_out = {'batadv': json_graph.node_link_data(batadv_graph), 'version': GRAPH_VERSION} - with open(tmp_graph_fn, 'w') as f: + with open(graph_fn, 'w') as f: json.dump(graph_out, f) - with open(tmp_nodelist_fn, 'w') as f: + with open(nodelist_fn, 'w') as f: json.dump(export_nodelist(now, nodedb), f) - os.rename(tmp_nodes_fn, nodes_fn) - os.rename(tmp_graph_fn, graph_fn) - os.rename(tmp_nodelist_fn, nodelist_fn) - # optional rrd graphs (trigger with --rrd) if params['rrd']: script_directory = os.path.dirname(os.path.realpath(__file__)) @@ -167,6 +157,7 @@ def main(params): rrd.update_database(nodedb['nodes']) rrd.update_images() + if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/gateway.json b/gateway.json deleted file mode 100644 index 4a72859..0000000 --- a/gateway.json +++ /dev/null @@ -1,29 +0,0 @@ -[ - { - "node_id": "deadbfff0101", - "hostname": "gw01" - }, - { - "node_id": "deadbeef0505", - "hostname": "gw02.hamburg.freifunk.net", - "network": { - "mac": "de:ad:be:ef:05:05", - "mesh": { - "bat0": { - "interfaces": { - "tunnel": [ - "de:ad:be:ff:05:05", - "de:ad:be:fc:05:05", - "de:ad:bf:ff:05:05" - ] - } - } - } - } - }, - { - "node_id": "00163efb9d8d", - "hostname": "gw03" - } -] - diff --git a/generate_aliases.py b/generate_aliases.py deleted file mode 100755 index 98fe407..0000000 --- a/generate_aliases.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python2 - -from __future__ import print_function - -import json -import os -import sys - -if len(sys.argv) != 2: - print('usage: ' + sys.argv[0] + ' /path/to/peers') - sys.exit(1) - -peersDir = sys.argv[1] - -def normalizeMac(mac): - mac = mac.lower() - normalized = '' - - n = 0 - - for c in mac: - if c != ':': - if n > 0 and n % 2 == 0: - normalized = normalized + ':' - normalized = normalized + c - n += 1 - - return normalized - -def toAlias(peer): - alias = {} - - if not (peer.has_key('name') and peer.has_key('mac')): - return None - - name = peer['name'] - mac = peer['mac'] - - alias['node_id'] = mac.replace(':', '') - alias['hostname'] = name - - if peer.has_key('geo'): - geo = peer['geo'] - - location = {} - - if geo.has_key('lon'): location['longitude'] = geo['lon'] - if geo.has_key('lat'): location['latitude'] = geo['lat'] - - alias['location'] = location - - #alias['network'] = {} - #alias['network']['mesh_interfaces'] = [mac] - - return alias - -aliases = [] - -for filename in os.listdir(peersDir): - if len(filename) == 0 or filename[0] == '.': - continue - - isGateway = False - - absFilename = peersDir + '/' + filename - if os.path.isfile(absFilename): - peerFile = open(absFilename, 'r') - try: - peerLines = peerFile.readlines() - peer = {} - - for line in peerLines: - parts = line.split() - - if len(parts) > 2: - if parts[1] == 'Knotenname:': - peer['name'] = parts[2] - - elif parts[0] == 'remote': - isGateway = True - - elif parts[1] == 'MAC:': - peer['mac'] = normalizeMac(parts[2]) - - elif parts[1] == 'Koordinaten:' and len(parts) > 3: - try: - peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])} - - except ValueError: - print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr) - - elif len(parts) == 2 and parts[0] == 'key': - keyParts = parts[1].split('"') - if len(keyParts) > 1: - peer['vpn'] = keyParts[1].lower() - - if isGateway: - continue - - alias = toAlias(peer) - if alias: - aliases.append(alias) - - except Exception as e: - print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr) - - finally: - peerFile.close() - -print(json.dumps(aliases)) diff --git a/generate_aliases_v2.py b/generate_aliases_v2.py deleted file mode 100755 index 7a04c7c..0000000 --- a/generate_aliases_v2.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python2 - -from __future__ import print_function - -import json -import os -import sys - -if len(sys.argv) != 2: - print('usage: ' + sys.argv[0] + ' /path/to/peers') - sys.exit(1) - -peersDir = sys.argv[1] - -def normalizeMac(mac): - mac = mac.lower() - normalized = '' - - n = 0 - - for c in mac: - if c != ':': - if n > 0 and n % 2 == 0: - normalized = normalized + ':' - normalized = normalized + c - n += 1 - - return normalized - -def toAlias(peer): - alias = {} - - if not (peer.has_key('name') and peer.has_key('mac')): - return None - - name = peer['name'] - mac = peer['mac'] - - alias['node_id'] = mac.replace(':', '') - alias['hostname'] = name - - if peer.has_key('geo'): - geo = peer['geo'] - - location = {} - - if geo.has_key('lon'): location['longitude'] = geo['lon'] - if geo.has_key('lat'): location['latitude'] = geo['lat'] - - alias['location'] = location - - #alias['network'] = {} - #alias['network']['mesh_interfaces'] = [mac] - - return {'nodeinfo':alias} - -aliases = {} - -for filename in os.listdir(peersDir): - if len(filename) == 0 or filename[0] == '.': - continue - - isGateway = False - - absFilename = peersDir + '/' + filename - if os.path.isfile(absFilename): - peerFile = open(absFilename, 'r') - try: - peerLines = peerFile.readlines() - peer = {} - - for line in peerLines: - parts = line.split() - - if len(parts) > 2: - if parts[1] == 'Knotenname:': - peer['name'] = parts[2] - - elif parts[0] == 'remote': - isGateway = True - - elif parts[1] == 'MAC:': - peer['mac'] = normalizeMac(parts[2]) - - elif parts[1] == 'Koordinaten:' and len(parts) > 3: - try: - peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])} - - except ValueError: - print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr) - - elif len(parts) == 2 and parts[0] == 'key': - keyParts = parts[1].split('"') - if len(keyParts) > 1: - peer['vpn'] = keyParts[1].lower() - - if isGateway: - continue - - alias = toAlias(peer) - if alias: - tmpid = alias['nodeinfo']['node_id'] -# alias['nodeinfo'].pop('node_id') - aliases[tmpid] = alias - - except Exception as e: - print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr) - - finally: - peerFile.close() - -print(json.dumps(aliases)) diff --git a/lib/alfred.py b/lib/alfred.py index 531eaea..4353874 100644 --- a/lib/alfred.py +++ b/lib/alfred.py @@ -13,7 +13,7 @@ class Alfred(object): raise RuntimeError('alfred: invalid unix socket path given') def _fetch(self, data_type): - cmd = ['/usr/local/bin/alfred-json', + cmd = ['alfred-json', '-z', '-f', 'json', '-r', str(data_type)] diff --git a/lib/batman.py b/lib/batman.py index ee0198b..ee0a0fe 100644 --- a/lib/batman.py +++ b/lib/batman.py @@ -96,3 +96,7 @@ if __name__ == "__main__": bc = Batman() vd = bc.vis_data() gw = bc.gateway_list() + for x in vd: + print(x) + print(gw) + print(bc.gateway_mode()) diff --git a/lib/graph.py b/lib/graph.py index 2833e6e..db1259e 100644 --- a/lib/graph.py +++ b/lib/graph.py @@ -25,6 +25,7 @@ def mark_vpn(graph, vpn_macs): components = map(frozenset, nx.weakly_connected_components(graph)) components = filter(vpn_macs.intersection, components) nodes = reduce(lambda a, b: a | b, components, set()) + for node in nodes: for k, v in graph[node].items(): v['vpn'] = True diff --git a/lib/nodelist.py b/lib/nodelist.py index a931dcf..15aea63 100644 --- a/lib/nodelist.py +++ b/lib/nodelist.py @@ -13,9 +13,6 @@ def export_nodelist(now, nodedb): node_out["status"] = dict() node_out["status"]["online"] = node["flags"]["online"] - if "firstseen" in node: - node_out["status"]["firstcontact"] = node["firstseen"] - if "lastseen" in node: node_out["status"]["lastcontact"] = node["lastseen"] diff --git a/lib/nodes.py b/lib/nodes.py index 86ce5b9..3c0fa2f 100644 --- a/lib/nodes.py +++ b/lib/nodes.py @@ -6,10 +6,6 @@ from functools import reduce def build_mac_table(nodes): macs = dict() for node_id, node in nodes.items(): - try: - macs[node['network']['mac']] = node_id - except KeyError: - pass try: for mac in node['nodeinfo']['network']['mesh_interfaces']: macs[mac] = node_id @@ -27,11 +23,6 @@ def build_mac_table(nodes): macs[mac] = node_id except KeyError: pass - try: - for mac in node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']: - macs[mac] = node_id - except KeyError: - pass try: for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']: @@ -65,23 +56,12 @@ def mark_online(node, now): node['flags']['online'] = True -def overrideFields(dest, src, fields): - for field in fields: - if field in src: - dest[field] = src[field] - else: - dest.pop(field, None) - - -def import_nodeinfo(nodes, nodeinfos, now, assume_online=False, statics=False): +def import_nodeinfo(nodes, nodeinfos, now, assume_online=False): for nodeinfo in filter(lambda d: 'node_id' in d, nodeinfos): - node = nodes.setdefault(nodeinfo['node_id'], {'flags': {'online': False, 'gateway': False}}) - - if statics: - node['nodeinfo'] = node.setdefault('nodeinfo', {}) - overrideFields(node['nodeinfo'], nodeinfo, ['hostname', 'location', 'node_id']) - else: - node['nodeinfo'] = nodeinfo + node = nodes.setdefault(nodeinfo['node_id'], {'flags': dict()}) + node['nodeinfo'] = nodeinfo + node['flags']['online'] = False + node['flags']['gateway'] = False if assume_online: mark_online(node, now) @@ -154,11 +134,6 @@ def import_mesh_ifs_vis_data(nodes, vis_data): except KeyError: pass - try: - ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel'])) - except KeyError: - pass - try: ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other'])) except KeyError: @@ -180,6 +155,7 @@ def import_vis_clientcount(nodes, vis_data): def mark_gateways(nodes, gateways): macs = build_mac_table(nodes) gateways = filter(lambda d: d in macs, gateways) + for node in map(lambda d: nodes[macs[d]], gateways): node['flags']['gateway'] = True diff --git a/mkmap.sh b/mkmap.sh deleted file mode 100755 index 9423943..0000000 --- a/mkmap.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -FFMAPPATH='/opt/ffmap-backend/' -PEERS="/etc/fastd/ffhh-mesh-vpn/peers" - -python2 $FFMAPPATH/generate_aliases.py $PEERS > $FFMAPPATH/aliases.json -#python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 --vpn de:ad:be:ff:05:05 --vpn de:ad:be:ff:05:06 --vpn de:ad:be:ff:03:03 --vpn de:ad:be:ff:22:22 --vpn de:ad:be:ff:22:23 --vpn de:ad:be:ff:88:88 --vpn de:ad:be:ff:88:89 --vpn de:ad:bf:ff:88:88 --vpn de:ad:bf:ff:22:22 --vpn de:ad:bf:ff:03:03 --vpn de:ad:bf:ff:05:05 --vpn de:ad:bf:ff:01:01 --vpn de:ad:be:fc:03:03 --vpn 00:16:3e:53:75:0d --vpn de:ad:be:fc:05:05 --vpn de:ad:be:fc:01:01 --vpn de:ad:be:ef:03:03 --vpn de:ad:be:ef:01:01 --vpn de:ad:be:ef:05:05 --vpn 00:16:3e:fb:9d:8d --vpn 00:16:3e:fb:9d:9d -python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 de:ad:be:ff:05:05 de:ad:be:ff:05:06 de:ad:be:ff:03:03 de:ad:be:ff:22:22 de:ad:be:ff:22:23 de:ad:be:ff:88:88 de:ad:be:ff:88:89 de:ad:bf:ff:88:88 de:ad:bf:ff:22:22 de:ad:bf:ff:03:03 de:ad:bf:ff:05:05 de:ad:bf:ff:01:01 de:ad:be:fc:03:03 00:16:3e:53:75:0d de:ad:be:fc:05:05 de:ad:be:fc:01:01 de:ad:be:ef:03:03 de:ad:be:ef:01:01 de:ad:be:ef:05:05 00:16:3e:fb:9d:8d 00:16:3e:fb:9d:9d diff --git a/node_number.py b/node_number.py deleted file mode 100755 index 20cd00e..0000000 --- a/node_number.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -#Bibliotheken importieren -import time -import datetime -import json -import urllib2 - -#Datei oeffnen -Datei = urllib2.urlopen('https://map.hamburg.freifunk.net/nodes.json') -Datei_Sued = urllib2.urlopen('https://map.hamburg.freifunk.net/hhsued/mv1/nodes.json') - -Text = Datei.read() -Knotenzahl = Text.count('"online": true') -Text = Datei_Sued.read() -Knotenzahl = Knotenzahl + Text.count('"online":true') - -#Zeit holen -thetime = datetime.datetime.now().isoformat() - -ffhh = None - -#Freifunk API-Datei einladen und JSON lesen -with open('/var/www/meta/ffhh.json', 'r') as fp: - ffhh = json.load(fp) - -#Attribute Zeitstempel und Knotenanzahl setzen -ffhh['state']['lastchange'] = thetime -ffhh['state']['nodes'] = Knotenzahl - -#Freifunk API-Datein mit geaenderten werten schreiben -with open('/var/www/meta/ffhh.json', 'w') as fp: - json.dump(ffhh, fp, indent=2, separators=(',', ': '))