Compare commits

..

No commits in common. "dev" and "feature-ffmap-d3-jq" have entirely different histories.

12 changed files with 20 additions and 384 deletions

View file

@ -1,42 +0,0 @@
#!/usr/bin/env python3
import subprocess
import json
from collections import MutableMapping
def rec_merge(d1, d2):
'''
Update two dicts of dicts recursively,
if either mapping has leaves that are non-dicts,
the second's leaf overwrites the first's.
'''
for k, v in d1.items(): # in Python 2, use .iteritems()!
if k in d2:
# this next check is the only difference!
if all(isinstance(e, MutableMapping) for e in (v, d2[k])):
d2[k] = rec_merge(v, d2[k])
# we could further check types and merge as appropriate here.
d3 = d1.copy()
d3.update(d2)
return d3
class alfred_merge:
def __init__(self,request_data_type_1 = 158, request_data_type_2 = 159):
self.request_data_type_1 = request_data_type_1
self.request_data_type_2 = request_data_type_2
def aliases(self):
output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_1),"-f","json"])
alfred_data_1 = json.loads(output.decode("utf-8"))
output = subprocess.check_output(["/usr/local/bin/alfred-json","-z", "-r",str(self.request_data_type_2),"-f","json"])
alfred_data_2 = json.loads(output.decode("utf-8"))
return json.dumps(rec_merge(alfred_data_1, alfred_data_2))
if __name__ == "__main__":
ad = alfred_merge()
al = ad.aliases()
print(al)

View file

@ -27,13 +27,8 @@ def main(params):
os.makedirs(params['dest_dir'], exist_ok=True) os.makedirs(params['dest_dir'], exist_ok=True)
nodes_fn = os.path.join(params['dest_dir'], 'nodes.json') nodes_fn = os.path.join(params['dest_dir'], 'nodes.json')
tmp_nodes_fn = os.path.join(params['dest_dir'], 'nodes.json.tmp')
graph_fn = os.path.join(params['dest_dir'], 'graph.json') graph_fn = os.path.join(params['dest_dir'], 'graph.json')
tmp_graph_fn = os.path.join(params['dest_dir'], 'graph.json.tmp')
nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json') nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json')
tmp_nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json.tmp')
now = datetime.utcnow().replace(microsecond=0) now = datetime.utcnow().replace(microsecond=0)
@ -89,9 +84,9 @@ def main(params):
# integrate static aliases data # integrate static aliases data
for aliases in params['aliases']: for aliases in params['aliases']:
with open(aliases, 'r') as f: with open(aliases, 'r') as f:
# nodeinfo = validate_nodeinfos(json.load(f)) nodeinfo = validate_nodeinfos(json.load(f))
nodes.import_nodeinfo(nodedb['nodes'], json.load(f), nodes.import_nodeinfo(nodedb['nodes'], nodeinfo,
now, assume_online=False, statics=True) now, assume_online=False)
nodes.reset_statistics(nodedb['nodes']) nodes.reset_statistics(nodedb['nodes'])
for alfred in alfred_instances: for alfred in alfred_instances:
@ -102,6 +97,7 @@ def main(params):
for batman in batman_instances: for batman in batman_instances:
vd = batman.vis_data() vd = batman.vis_data()
gwl = batman.gateway_list() gwl = batman.gateway_list()
mesh_info.append((vd, gwl)) mesh_info.append((vd, gwl))
# update nodedb from batman-adv data # update nodedb from batman-adv data
@ -130,8 +126,6 @@ def main(params):
try: try:
for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]: for mac in node["nodeinfo"]["network"]["mesh"]["bat0"]["interfaces"]["tunnel"]:
macs.add(mac) macs.add(mac)
for mac in node["nodeinfo"]["network"]["mesh"]["bat-ffhh"]["interfaces"]["tunnel"]:
macs.add(mac)
except KeyError: except KeyError:
pass pass
@ -143,22 +137,18 @@ def main(params):
batadv_graph = graph.to_undirected(batadv_graph) batadv_graph = graph.to_undirected(batadv_graph)
# write processed data to dest dir # write processed data to dest dir
with open(tmp_nodes_fn, 'w') as f: with open(nodes_fn, 'w') as f:
json.dump(nodedb, f) json.dump(nodedb, f)
graph_out = {'batadv': json_graph.node_link_data(batadv_graph), graph_out = {'batadv': json_graph.node_link_data(batadv_graph),
'version': GRAPH_VERSION} 'version': GRAPH_VERSION}
with open(tmp_graph_fn, 'w') as f: with open(graph_fn, 'w') as f:
json.dump(graph_out, f) json.dump(graph_out, f)
with open(tmp_nodelist_fn, 'w') as f: with open(nodelist_fn, 'w') as f:
json.dump(export_nodelist(now, nodedb), f) json.dump(export_nodelist(now, nodedb), f)
os.rename(tmp_nodes_fn, nodes_fn)
os.rename(tmp_graph_fn, graph_fn)
os.rename(tmp_nodelist_fn, nodelist_fn)
# optional rrd graphs (trigger with --rrd) # optional rrd graphs (trigger with --rrd)
if params['rrd']: if params['rrd']:
script_directory = os.path.dirname(os.path.realpath(__file__)) script_directory = os.path.dirname(os.path.realpath(__file__))
@ -167,6 +157,7 @@ def main(params):
rrd.update_database(nodedb['nodes']) rrd.update_database(nodedb['nodes'])
rrd.update_images() rrd.update_images()
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()

View file

@ -1,29 +0,0 @@
[
{
"node_id": "deadbfff0101",
"hostname": "gw01"
},
{
"node_id": "deadbeef0505",
"hostname": "gw02.hamburg.freifunk.net",
"network": {
"mac": "de:ad:be:ef:05:05",
"mesh": {
"bat0": {
"interfaces": {
"tunnel": [
"de:ad:be:ff:05:05",
"de:ad:be:fc:05:05",
"de:ad:bf:ff:05:05"
]
}
}
}
}
},
{
"node_id": "00163efb9d8d",
"hostname": "gw03"
}
]

View file

@ -1,110 +0,0 @@
#!/usr/bin/env python2
from __future__ import print_function
import json
import os
import sys
if len(sys.argv) != 2:
print('usage: ' + sys.argv[0] + ' /path/to/peers')
sys.exit(1)
peersDir = sys.argv[1]
def normalizeMac(mac):
mac = mac.lower()
normalized = ''
n = 0
for c in mac:
if c != ':':
if n > 0 and n % 2 == 0:
normalized = normalized + ':'
normalized = normalized + c
n += 1
return normalized
def toAlias(peer):
alias = {}
if not (peer.has_key('name') and peer.has_key('mac')):
return None
name = peer['name']
mac = peer['mac']
alias['node_id'] = mac.replace(':', '')
alias['hostname'] = name
if peer.has_key('geo'):
geo = peer['geo']
location = {}
if geo.has_key('lon'): location['longitude'] = geo['lon']
if geo.has_key('lat'): location['latitude'] = geo['lat']
alias['location'] = location
#alias['network'] = {}
#alias['network']['mesh_interfaces'] = [mac]
return alias
aliases = []
for filename in os.listdir(peersDir):
if len(filename) == 0 or filename[0] == '.':
continue
isGateway = False
absFilename = peersDir + '/' + filename
if os.path.isfile(absFilename):
peerFile = open(absFilename, 'r')
try:
peerLines = peerFile.readlines()
peer = {}
for line in peerLines:
parts = line.split()
if len(parts) > 2:
if parts[1] == 'Knotenname:':
peer['name'] = parts[2]
elif parts[0] == 'remote':
isGateway = True
elif parts[1] == 'MAC:':
peer['mac'] = normalizeMac(parts[2])
elif parts[1] == 'Koordinaten:' and len(parts) > 3:
try:
peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
except ValueError:
print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
elif len(parts) == 2 and parts[0] == 'key':
keyParts = parts[1].split('"')
if len(keyParts) > 1:
peer['vpn'] = keyParts[1].lower()
if isGateway:
continue
alias = toAlias(peer)
if alias:
aliases.append(alias)
except Exception as e:
print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
finally:
peerFile.close()
print(json.dumps(aliases))

View file

@ -1,112 +0,0 @@
#!/usr/bin/env python2
from __future__ import print_function
import json
import os
import sys
if len(sys.argv) != 2:
print('usage: ' + sys.argv[0] + ' /path/to/peers')
sys.exit(1)
peersDir = sys.argv[1]
def normalizeMac(mac):
mac = mac.lower()
normalized = ''
n = 0
for c in mac:
if c != ':':
if n > 0 and n % 2 == 0:
normalized = normalized + ':'
normalized = normalized + c
n += 1
return normalized
def toAlias(peer):
alias = {}
if not (peer.has_key('name') and peer.has_key('mac')):
return None
name = peer['name']
mac = peer['mac']
alias['node_id'] = mac.replace(':', '')
alias['hostname'] = name
if peer.has_key('geo'):
geo = peer['geo']
location = {}
if geo.has_key('lon'): location['longitude'] = geo['lon']
if geo.has_key('lat'): location['latitude'] = geo['lat']
alias['location'] = location
#alias['network'] = {}
#alias['network']['mesh_interfaces'] = [mac]
return {'nodeinfo':alias}
aliases = {}
for filename in os.listdir(peersDir):
if len(filename) == 0 or filename[0] == '.':
continue
isGateway = False
absFilename = peersDir + '/' + filename
if os.path.isfile(absFilename):
peerFile = open(absFilename, 'r')
try:
peerLines = peerFile.readlines()
peer = {}
for line in peerLines:
parts = line.split()
if len(parts) > 2:
if parts[1] == 'Knotenname:':
peer['name'] = parts[2]
elif parts[0] == 'remote':
isGateway = True
elif parts[1] == 'MAC:':
peer['mac'] = normalizeMac(parts[2])
elif parts[1] == 'Koordinaten:' and len(parts) > 3:
try:
peer['geo'] = {'lat': float(parts[2]), 'lon': float(parts[3])}
except ValueError:
print('Error in %s: Invalid coordinates: %s' % (absFilename, parts[2:4]), file = sys.stderr)
elif len(parts) == 2 and parts[0] == 'key':
keyParts = parts[1].split('"')
if len(keyParts) > 1:
peer['vpn'] = keyParts[1].lower()
if isGateway:
continue
alias = toAlias(peer)
if alias:
tmpid = alias['nodeinfo']['node_id']
# alias['nodeinfo'].pop('node_id')
aliases[tmpid] = alias
except Exception as e:
print('Error in %s, ignoring peer: %s' % (absFilename, e), file = sys.stderr)
finally:
peerFile.close()
print(json.dumps(aliases))

View file

@ -13,7 +13,7 @@ class Alfred(object):
raise RuntimeError('alfred: invalid unix socket path given') raise RuntimeError('alfred: invalid unix socket path given')
def _fetch(self, data_type): def _fetch(self, data_type):
cmd = ['/usr/local/bin/alfred-json', cmd = ['alfred-json',
'-z', '-z',
'-f', 'json', '-f', 'json',
'-r', str(data_type)] '-r', str(data_type)]

View file

@ -96,3 +96,7 @@ if __name__ == "__main__":
bc = Batman() bc = Batman()
vd = bc.vis_data() vd = bc.vis_data()
gw = bc.gateway_list() gw = bc.gateway_list()
for x in vd:
print(x)
print(gw)
print(bc.gateway_mode())

View file

@ -25,6 +25,7 @@ def mark_vpn(graph, vpn_macs):
components = map(frozenset, nx.weakly_connected_components(graph)) components = map(frozenset, nx.weakly_connected_components(graph))
components = filter(vpn_macs.intersection, components) components = filter(vpn_macs.intersection, components)
nodes = reduce(lambda a, b: a | b, components, set()) nodes = reduce(lambda a, b: a | b, components, set())
for node in nodes: for node in nodes:
for k, v in graph[node].items(): for k, v in graph[node].items():
v['vpn'] = True v['vpn'] = True

View file

@ -13,9 +13,6 @@ def export_nodelist(now, nodedb):
node_out["status"] = dict() node_out["status"] = dict()
node_out["status"]["online"] = node["flags"]["online"] node_out["status"]["online"] = node["flags"]["online"]
if "firstseen" in node:
node_out["status"]["firstcontact"] = node["firstseen"]
if "lastseen" in node: if "lastseen" in node:
node_out["status"]["lastcontact"] = node["lastseen"] node_out["status"]["lastcontact"] = node["lastseen"]

View file

@ -6,10 +6,6 @@ from functools import reduce
def build_mac_table(nodes): def build_mac_table(nodes):
macs = dict() macs = dict()
for node_id, node in nodes.items(): for node_id, node in nodes.items():
try:
macs[node['network']['mac']] = node_id
except KeyError:
pass
try: try:
for mac in node['nodeinfo']['network']['mesh_interfaces']: for mac in node['nodeinfo']['network']['mesh_interfaces']:
macs[mac] = node_id macs[mac] = node_id
@ -27,11 +23,6 @@ def build_mac_table(nodes):
macs[mac] = node_id macs[mac] = node_id
except KeyError: except KeyError:
pass pass
try:
for mac in node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']:
macs[mac] = node_id
except KeyError:
pass
try: try:
for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']: for mac in node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']:
@ -65,23 +56,12 @@ def mark_online(node, now):
node['flags']['online'] = True node['flags']['online'] = True
def overrideFields(dest, src, fields): def import_nodeinfo(nodes, nodeinfos, now, assume_online=False):
for field in fields:
if field in src:
dest[field] = src[field]
else:
dest.pop(field, None)
def import_nodeinfo(nodes, nodeinfos, now, assume_online=False, statics=False):
for nodeinfo in filter(lambda d: 'node_id' in d, nodeinfos): for nodeinfo in filter(lambda d: 'node_id' in d, nodeinfos):
node = nodes.setdefault(nodeinfo['node_id'], {'flags': {'online': False, 'gateway': False}}) node = nodes.setdefault(nodeinfo['node_id'], {'flags': dict()})
node['nodeinfo'] = nodeinfo
if statics: node['flags']['online'] = False
node['nodeinfo'] = node.setdefault('nodeinfo', {}) node['flags']['gateway'] = False
overrideFields(node['nodeinfo'], nodeinfo, ['hostname', 'location', 'node_id'])
else:
node['nodeinfo'] = nodeinfo
if assume_online: if assume_online:
mark_online(node, now) mark_online(node, now)
@ -112,7 +92,6 @@ def import_statistics(nodes, stats):
add(node, stats, 'memory_usage', ['memory'], add(node, stats, 'memory_usage', ['memory'],
lambda d: 1 - d['free'] / d['total']) lambda d: 1 - d['free'] / d['total'])
add(node, stats, 'rootfs_usage', ['rootfs_usage']) add(node, stats, 'rootfs_usage', ['rootfs_usage'])
add(node, stats, 'traffic', ['traffic'])
def import_mesh_ifs_vis_data(nodes, vis_data): def import_mesh_ifs_vis_data(nodes, vis_data):
@ -154,11 +133,6 @@ def import_mesh_ifs_vis_data(nodes, vis_data):
except KeyError: except KeyError:
pass pass
try:
ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat-ffhh']['interfaces']['tunnel']))
except KeyError:
pass
try: try:
ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other'])) ifs = ifs.union(set(node['nodeinfo']['network']['mesh']['bat0']['interfaces']['other']))
except KeyError: except KeyError:
@ -180,6 +154,7 @@ def import_vis_clientcount(nodes, vis_data):
def mark_gateways(nodes, gateways): def mark_gateways(nodes, gateways):
macs = build_mac_table(nodes) macs = build_mac_table(nodes)
gateways = filter(lambda d: d in macs, gateways) gateways = filter(lambda d: d in macs, gateways)
for node in map(lambda d: nodes[macs[d]], gateways): for node in map(lambda d: nodes[macs[d]], gateways):
node['flags']['gateway'] = True node['flags']['gateway'] = True

View file

@ -1,7 +0,0 @@
#!/bin/bash
FFMAPPATH='/opt/ffmap-backend/'
PEERS="/etc/fastd/ffhh-mesh-vpn/peers"
python2 $FFMAPPATH/generate_aliases.py $PEERS > $FFMAPPATH/aliases.json
#python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 --vpn de:ad:be:ff:05:05 --vpn de:ad:be:ff:05:06 --vpn de:ad:be:ff:03:03 --vpn de:ad:be:ff:22:22 --vpn de:ad:be:ff:22:23 --vpn de:ad:be:ff:88:88 --vpn de:ad:be:ff:88:89 --vpn de:ad:bf:ff:88:88 --vpn de:ad:bf:ff:22:22 --vpn de:ad:bf:ff:03:03 --vpn de:ad:bf:ff:05:05 --vpn de:ad:bf:ff:01:01 --vpn de:ad:be:fc:03:03 --vpn 00:16:3e:53:75:0d --vpn de:ad:be:fc:05:05 --vpn de:ad:be:fc:01:01 --vpn de:ad:be:ef:03:03 --vpn de:ad:be:ef:01:01 --vpn de:ad:be:ef:05:05 --vpn 00:16:3e:fb:9d:8d --vpn 00:16:3e:fb:9d:9d
python3 $FFMAPPATH/backend.py -d /var/www/meshviewer/ --aliases $FFMAPPATH/aliases.json $FFMAPPATH/gateway.json -m bat0:/var/run/alfred.sock -p 30 --vpn de:ad:be:ff:01:01 de:ad:be:ff:05:05 de:ad:be:ff:05:06 de:ad:be:ff:03:03 de:ad:be:ff:22:22 de:ad:be:ff:22:23 de:ad:be:ff:88:88 de:ad:be:ff:88:89 de:ad:bf:ff:88:88 de:ad:bf:ff:22:22 de:ad:bf:ff:03:03 de:ad:bf:ff:05:05 de:ad:bf:ff:01:01 de:ad:be:fc:03:03 00:16:3e:53:75:0d de:ad:be:fc:05:05 de:ad:be:fc:01:01 de:ad:be:ef:03:03 de:ad:be:ef:01:01 de:ad:be:ef:05:05 00:16:3e:fb:9d:8d 00:16:3e:fb:9d:9d

View file

@ -1,32 +0,0 @@
#!/usr/bin/env python
#Bibliotheken importieren
import time
import datetime
import json
import urllib2
#Datei oeffnen
Datei = urllib2.urlopen('https://map.hamburg.freifunk.net/nodes.json')
Datei_Sued = urllib2.urlopen('https://map.hamburg.freifunk.net/hhsued/mv1/nodes.json')
Text = Datei.read()
Knotenzahl = Text.count('"online": true')
Text = Datei_Sued.read()
Knotenzahl = Knotenzahl + Text.count('"online":true')
#Zeit holen
thetime = datetime.datetime.now().isoformat()
ffhh = None
#Freifunk API-Datei einladen und JSON lesen
with open('/var/www/meta/ffhh.json', 'r') as fp:
ffhh = json.load(fp)
#Attribute Zeitstempel und Knotenanzahl setzen
ffhh['state']['lastchange'] = thetime
ffhh['state']['nodes'] = Knotenzahl
#Freifunk API-Datein mit geaenderten werten schreiben
with open('/var/www/meta/ffhh.json', 'w') as fp:
json.dump(ffhh, fp, indent=2, separators=(',', ': '))