Compare commits

..

No commits in common. "master" and "meshviewer" have entirely different histories.

13 changed files with 126 additions and 203 deletions

View file

@ -1,8 +1,7 @@
#!/usr/bin/python3
import argparse
# from parser.NodesParser import NodesParser
# from parser.GraphParser import GraphParser
from parser.Hopglass import Hopglass
from parser.NodesParser import NodesParser
from parser.GraphParser import GraphParser
from cloud.Node import Node
from cloud.Link import Link
from cloud.GlobalGraph import GlobalGraph
@ -14,9 +13,8 @@ from info.Info import Info
class NodeHierarchy(object):
def __init__(self):
self.__args__ = self.__parseArguments__()
self.__hopglass = Hopglass(self.__args__.raw_json)
# self.__nodesJson__ = NodesParser(self.__args__.json_path)
# self.__graphJson__ = GraphParser(self.__args__.json_path)
self.__nodesJson__ = NodesParser(self.__args__.json_path)
self.__graphJson__ = GraphParser(self.__args__.json_path)
self.__shapesJson__ = self.__parseShapes__()
self.nodes = self.__createNodeObjects__()
self.links = self.__createLinkObjects__()
@ -42,29 +40,26 @@ class NodeHierarchy(object):
def __createNodeObjects__(self):
nodes = {}
for nodeID, nodeValue in self.__hopglass.nodes.items():
if nodeValue['nodeinfo']['node_id']:
print('Create Node object #',len(nodes), '\r',end = '')
nodes[nodeID] = Node(self.__prepareNodeData__(nodeValue))
for nodeID, nodeValue in self.__nodesJson__.nodes.items():
print('Create Node object #',len(nodes), '\r',end = '')
nodes[nodeID] = Node(nodeValue)
print('')
return nodes
def __prepareNodeData__(self, nodeValue):
if self.__args__.site_to_target_prefix:
pref = self.__args__.site_to_target_prefix.split(',')
try:
nodeValue['nodeinfo']['system']['site_code'] = nodeValue['nodeinfo']['system']['site_code'].replace(pref[0],pref[1])
except:
pass
return nodeValue
def __createLinkObjects__(self):
links = []
for linkParID, linkPar in self.__hopglass.links.items():
for linkID, link in linkPar.items():
print('Create Link object #',len(links), '\r',end = '')
if linkParID[0] != 'null' and linkParID[1] != 'null':
links.append(Link(link, (self.nodes[linkParID[0]], self.nodes[linkParID[1]])))
for link in self.__graphJson__.links:
try:
srcNode = self.nodes[link['source']['node_id']]
except:
srcNode = None
try:
dstNode = self.nodes[link['target']['node_id']]
except:
dstNode = None
print('Create Link object #',len(links), '\r',end = '')
links.append(Link(link, srcNode, dstNode))
print('')
return links
@ -73,13 +68,12 @@ class NodeHierarchy(object):
def __parseArguments__(self):
parser = argparse.ArgumentParser(description='This Script generates a hierarchical nodes list for node migration using nginx geo feature.')
parser.add_argument('-r', '--raw-json', required=False, default='https://karte.freifunk-muensterland.de/data/raw.json', help='Location of raw.json file (can be local folder or remote URL).')
parser.add_argument('-j', '--json-path', required=False, default='https://service.freifunk-muensterland.de/maps/data/', help='Path of nodes.json and graph.json (can be local folder or remote URL).')
parser.add_argument('-s', '--shapes-path', required=False, default='https://freifunk-muensterland.de/md-fw-dl/shapes/', help='Path of shapefiles (can be local folder or remote URL).')
parser.add_argument('-t', '--targets', nargs='+', required=True, help='List of targets which should be proceeded. Example: -t citya cityb ...')
parser.add_argument('-sttp', '--site-to-target-prefix', required=False, help='Used to match site and target also when prefixes are different. Example: -sttp "ffmsd,domaene"')
parser.add_argument('-o', '--out-file', default='./webserver-configuration', required=False, help='Filename where the generated Output should stored.')
parser.add_argument('-v', '--debug', required=False, action='store_true', help='Enable debugging output.')
parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan', 'domain_transitions_only'), help='Filter out nodes and local clouds based on filter rules.')
parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan'), help='Filter out nodes and local clouds based on filter rules.')
parser.add_argument('-i', '--info', nargs='*', required=False, choices=('get_offline_nodes','offline'), help='Get infos about the graph, links and nodes.')
parser.add_argument('-if', '--info-filters', nargs='*', required=False, help='Filter info results. Currently supported: min_age:TIME_RANGE, max_age:TIME_RANGE. Examples: -if min_age:1d max_age:2w')
parser.add_argument('-iop', '--info-out-path', required=False, default='./', help='Folder where info files should be written. Default: ./')

View file

@ -1,5 +1,5 @@
# Node Hierarchy
Dieses Tool generiert auf Basis einer ``raw.json`` des [hopglass-server](https://github.com/hopglass/hopglass-server) sowie (Multi-)Polygonen (im [geojson](http://geojson.org/) Format) der einzelnen Zieldomänen eine [nginx](http://nginx.org/) Konfigurationsdatei (auf Basis des [Geo-Moduls](http://nginx.org/en/docs/http/ngx_http_geo_module.html)), um Knoten in der richtigen Reihenfolge umzuziehen.
Dieses Tool generiert auf Basis einer ``graph.json`` und ``nodes.json`` des [Meshviewers](https://github.com/ffnord/meshviewer/) sowie (Multi-)Polygonen (im [geojson](http://geojson.org/) Format) der einzelnen Zieldomänen eine [nginx](http://nginx.org/) Konfigurationsdatei (auf Basis des [Geo-Moduls](http://nginx.org/en/docs/http/ngx_http_geo_module.html)), um Knoten in der richtigen Reihenfolge umzuziehen.
## Vorgehensweise
@ -28,10 +28,9 @@ Die Hilfe liefert folgendes:
```
$ ./NodeHierarchy.py --help
usage: NodeHierarchy.py [-h] [-r RAW_JSON] [-s SHAPES_PATH] -t TARGETS
[TARGETS ...] [-sttp SITE_TO_TARGET_PREFIX]
[-o OUT_FILE] [-v]
[-f [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} ...]]]
usage: NodeHierarchy.py [-h] [-j JSON_PATH] [-s SHAPES_PATH] -t TARGETS
[TARGETS ...] [-o OUT_FILE] [-v]
[-f [{exclude_clouds_with_lan_links,no_lan} [{exclude_clouds_with_lan_links,no_lan} ...]]]
[-i [{get_offline_nodes,offline} [{get_offline_nodes,offline} ...]]]
[-if [INFO_FILTERS [INFO_FILTERS ...]]]
[-iop INFO_OUT_PATH]
@ -42,22 +41,19 @@ geo feature.
optional arguments:
-h, --help show this help message and exit
-r RAW_JSON, --raw-json RAW_JSON
Location of raw.json file (can be local folder or
remote URL).
-j JSON_PATH, --json-path JSON_PATH
Path of nodes.json and graph.json (can be local folder
or remote URL).
-s SHAPES_PATH, --shapes-path SHAPES_PATH
Path of shapefiles (can be local folder or remote
URL).
-t TARGETS [TARGETS ...], --targets TARGETS [TARGETS ...]
List of targets which should be proceeded. Example: -t
citya cityb ...
-sttp SITE_TO_TARGET_PREFIX, --site-to-target-prefix SITE_TO_TARGET_PREFIX
Used to match site and target also when prefixes are
different. Example: -sttp "ffmsd,domaene"
-o OUT_FILE, --out-file OUT_FILE
Filename where the generated Output should stored.
-v, --debug Enable debugging output.
-f [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} ...]], --filters [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} ...]]
-f [{exclude_clouds_with_lan_links,no_lan} [{exclude_clouds_with_lan_links,no_lan} ...]], --filters [{exclude_clouds_with_lan_links,no_lan} [{exclude_clouds_with_lan_links,no_lan} ...]]
Filter out nodes and local clouds based on filter
rules.
-i [{get_offline_nodes,offline} [{get_offline_nodes,offline} ...]], --info [{get_offline_nodes,offline} [{get_offline_nodes,offline} ...]]
@ -76,7 +72,7 @@ optional arguments:
### Anmerkungen
- ``--targets`` Gibt die Namen der Ziele (Zieldomänen) an. Der Geo-Schalter in der nginx-Konfiguration wird ebenfalls diesen Namen tragen.
- ``--raw-json`` Gibt den Ort der raw.json (hopglass-server) an. Default: ``https://karte.freifunk-muensterland.de/data/raw.json``
- ``--json-path`` Gibt das Daten-Verzeichnis eures Meshviewers an. Default: ``https://service.freifunk-muensterland.de/maps/data/``
- ``--shapes-path`` Verzeichnis an dem die Shapefiles der einzelnen Ziel-Domänen liegen. Default: ``https://freifunk-muensterland.de/md-fw-dl/shapes/``
- *Anmerkung:* Es werden Dateien in Abhängigkeit mit den Target-Namen im Verzeichnis erwartet.
- *Beispiel:* Bei ``-targets domaene01 domaene02`` werden die Dateien ``domaene01.geojson`` und ``domaene02.geojson`` erwartet.
@ -93,8 +89,7 @@ Weitere Filterungen lassen sich über das ``--filters`` Attribut aktivieren.
Folgende Filter sind derzeit implementiert (zukünftig folgen noch weitere):
- ``exclude_clouds_with_lan_links`` bzw. ``no_lan`` filtert alle lokalen Wolken aus, in denen sich mindestens ein Mesh-on-LAN Link befindet
- ``domain_transitions_only`` filtert alle Knoten aus, die sich bereits in der richtigen Domäne befinden / die Firmware der richtigen Domäne besitzen
- ``exclude_clouds_with_lan_links`` bzw. ``no_lan`` Filtert alle lokalen Wolken aus, in denen sich mindestens ein Mesh-on-LAN Link befindet
## Nginx Konfiguration
@ -135,9 +130,9 @@ schreibt in die Datei ``./offline_nodes.csv`` (default-Einstellung der Schalter
## Bekannte Probleme
Wenn es sich bei der Quell-Domäne um eine L2TP-Domäne handelt, läuft das Tool derzeit nur, wenn [alfred](https://github.com/ffnord/ffnord-alfred-announce) oder respondd auf allen Gateway-Servern läuft.
Wenn es sich bei der Quell-Domäne um eine L2TP-Domäne handelt, läuft das Tool derzeit nur, wenn [alfred](https://github.com/ffnord/ffnord-alfred-announce) auf allen Gateway-Servern läuft.
*Anmerkung:* Wenn in der ``graph.json`` mehrere Domänen vorhanden sind und dort teilweise L2TP-Domänen vorhanden sind (dieses aber nicht das Gebiet eurer Zieldomäne betrifft), kann das sehr negative Auswirkungen auf die Laufzeit haben (> 30 Sekunden).
*Anmerkung:* Wenn in der ``nodes.json`` und ``graph.json`` mehrere Domänen vorhanden sind und dort teilweise L2TP-Domänen vorhanden sind (dieses aber nicht das Gebiet eurer Zieldomäne betrifft), kann das sehr negative Auswirkungen auf die Laufzeit haben (> 30 Sekunden).
## Lizenz

View file

@ -26,19 +26,18 @@ class GlobalGraph(Graph):
def __getConnectedNodes__(self, nodeID, trace = []):
neighNodeIDs = self.getNeighbourNodeIDsForNodeID(nodeID)
trace_new = list(set(trace + neighNodeIDs))
trace_new = trace[:] + [x for x in neighNodeIDs if x not in trace]
for neighNodeID in neighNodeIDs:
if neighNodeID not in trace:
trace_new = list(set(trace_new + self.__getConnectedNodes__(neighNodeID, trace_new)))
trace_new = trace_new + [x for x in self.__getConnectedNodes__(neighNodeID, trace_new) if x not in trace_new]
return trace_new
def __createLocalCloudByNodesList__(self, nodesIDList):
nodes = {}
links = []
for nodeID in nodesIDList:
if nodeID:
nodes[nodeID] = self.__nodes__[nodeID]
links = list(set(links + self.getLinksByNodeID(nodeID)))
nodes[nodeID] = self.__nodes__[nodeID]
links = links + [x for x in self.getLinksByNodeID(nodeID) if x not in links]
return LocalGraph(nodes, links, self.__enableDebugPrinting__)
def __debugPrint__(self):

View file

@ -26,7 +26,7 @@ class Graph(object):
if link.isVpn == False:
endpoints = link.getEndpointNodeIDs(getGateways = False)
if nodeID in endpoints:
neighNodeIDs = list(set(neighNodeIDs + endpoints))
neighNodeIDs = neighNodeIDs + [x for x in endpoints if x not in neighNodeIDs]
return neighNodeIDs
def getLinksByNodeID(self, nodeID):

View file

@ -1,31 +1,69 @@
class Link(object):
def __init__(self, LinkJsonObject, nodes):
self.__jsonObject = LinkJsonObject
self.linkType, self.isVpn = self.__getLinkType__()
self.__nodes = nodes
def __init__(self, LinkJsonObject, srcNode, dstNode):
self.__jsonObject__ = LinkJsonObject
self.__srcNode__ = srcNode
self.__dstNode__ = dstNode
self.linkType = self.__getLinkType__()
self.isVpn = self.__getLinkVpnState__()
def __getLinkType__(self):
types = [x['type'] for x in self.__jsonObject]
ltype = types[0]
lvpn = False
for x in types:
if x != 'unknown' and x != 'other':
if x == 'l2tp' or x == 'tunnel':
lvpn = True
val = x
return ltype, lvpn
def getEndpointNodes(self, getGateways = False):
return self.__nodes
def getEndpointNodeIDs(self, getGateways = True):
return [x.nodeID for x in self.__nodes]
def isNodeIDinLink(self, nodeID):
for x in self.__nodes:
if nodeID == x.nodeID:
type_src = None
type_dst = None
if self.__srcNode__ != None:
for k, v in self.__srcNode__.interfaces.items():
if self.__jsonObject__['source']['interface_mac'] in v:
type_src = k
if self.__dstNode__ != None:
for k, v in self.__dstNode__.interfaces.items():
if self.__jsonObject__['target']['interface_mac'] in v:
type_dst = k
if type_src == type_dst:
if type_src == None:
return 'unknown'
return type_src
else:
if type_src == None:
return type_dst
elif type_dst == None:
return type_src
else:
#print(self.__srcNode__.hostname, type_src, '<-->', self.__dstNode__.hostname, type_dst)
if type_src == 'wireless':
return type_dst
else:
return type_src
def __getLinkVpnState__(self):
if self.__jsonObject__['vpn'] == True:
return True
for node in self.getEndpointNodes(getGateways = True):
if node.isGateway == True:
return True
return False
def getEndpointNodes(self, getGateways = False):
nodes = []
if self.__srcNode__ != None:
if getGateways == True or self.__srcNode__.isGateway == False:
nodes.append(self.__srcNode__)
if self.__dstNode__ != None:
if getGateways == True or self.__dstNode__.isGateway == False:
nodes.append(self.__dstNode__)
return nodes
def getEndpointNodeIDs(self, getGateways = True):
nodeIDs = []
for node in self.getEndpointNodes(getGateways):
nodeIDs.append(node.nodeID)
return nodeIDs
def isNodeIDinLink(self, nodeID):
for endpoint in self.getEndpointNodes():
if endpoint.nodeID == nodeID:
return True
return False
def isNodeInLink(self, node):
return self.isNodeIDinLink(node.nodeID)

View file

@ -125,9 +125,6 @@ class LocalGraph(Graph):
print('BranchesThatExistsInCloud:', self.getBranchesThatExistsInCloud())
print('lan links in cloud:')
for link in self.getLanLinksInCloud():
hosts = link.getEndpointNodes()
if len(hosts) == 1:
print(' ', hosts.hostname, 'has unknown neighbour.')
else:
print(' ', hosts[0].hostname, '<--->', hosts[1].hostname)
if link.__srcNode__ != None and link.__dstNode__ != None:
print(' ', link.__srcNode__.hostname, '<--->', link.__dstNode__.hostname)
print('=====')

View file

@ -4,13 +4,13 @@ class NodeInit(object):
self.nodeID = self.__jsonObject__['nodeinfo']['node_id']
self.interfaces = self.__getInterfaces__()
self.hostname = self.__jsonObject__['nodeinfo']['hostname']
self.isGateway = self.__jsonObject__['nodeinfo']['isGateway']
self.isGateway = self.__jsonObject__['flags']['gateway']
self.geo = self.__getGeo__()
self.isAutoupdaterEnabled = self.__getAutoupdaterStatus__()
self.autoupdaterBranch = self.__getBranch__()
self.isOnline = self.__jsonObject__['nodeinfo']['isOnline']
self.isOnline = self.__jsonObject__['flags']['online']
self.publicIPv6Addresses = self.__getPublicAddresses__()
self.domName = self.__getSiteCode__()
self.domID = self.__getSiteCode__()
def __getInterfaces__(self):
try:
@ -19,34 +19,31 @@ class NodeInit(object):
return {}
def __getAutoupdaterStatus__(self):
try:
if 'autoupdater' in self.__jsonObject__['nodeinfo']['software']:
return self.__jsonObject__['nodeinfo']['software']['autoupdater']['enabled']
except:
else:
return False
def __getBranch__(self):
try:
if 'autoupdater' in self.__jsonObject__['nodeinfo']['software']:
return self.__jsonObject__['nodeinfo']['software']['autoupdater']['branch']
except:
else:
return None
def __getGeo__(self):
try:
return {
'lat' : self.__jsonObject__['nodeinfo']['location']['latitude'],
'lon' : self.__jsonObject__['nodeinfo']['location']['longitude']
}
except:
return None
geo = {}
if 'location' in self.__jsonObject__['nodeinfo'] and 'latitude' in self.__jsonObject__['nodeinfo']['location'] and 'longitude' in self.__jsonObject__['nodeinfo']['location']:
geo['lat'] = self.__jsonObject__['nodeinfo']['location']['latitude']
geo['lon'] = self.__jsonObject__['nodeinfo']['location']['longitude']
return geo
return None
def __getPublicAddresses__(self):
addresses = []
try:
if 'addresses' in self.__jsonObject__['nodeinfo']['network']:
for address in self.__jsonObject__['nodeinfo']['network']['addresses']:
if not address.startswith('fe80'):
addresses.append(address)
except:
pass
return addresses
def __getSiteCode__(self):

View file

@ -3,7 +3,7 @@ class Filter(object):
self.__args__ = args
self.__filters__ = self.__getFilters()
def filterLocalGraphs(self, domain, localGraphs):
def filterLocalGraphs(self, localGraphs):
filteredGraphs = []
for localGraph in localGraphs:
if localGraph.isAutoupdaterEnabledOnAllNodes() == False:
@ -21,14 +21,8 @@ class Filter(object):
def __getFilters(self):
return [] if self.__args__.filters == None else self.__args__.filters
def filterNodes(self, domain, nodes):
def filterNodes(self, nodes):
filteredNodes = []
for node in nodes:
if 'domain_transitions_only' in self.__filters__:
try:
if domain.name == node.domName:
continue
except:
pass
filteredNodes.append(node)
return filteredNodes

View file

@ -17,9 +17,9 @@ class NginxConfGen(object):
def __genDomain__(self, domain):
nodes = {}
for localGraph in self.__filter__.filterLocalGraphs(domain, domain.localGraphs):
for localGraph in self.__filter__.filterLocalGraphs(domain.localGraphs):
try:
for node in self.__filter__.filterNodes(domain, localGraph.getNodesWithNoDependencies()):
for node in self.__filter__.filterNodes(localGraph.getNodesWithNoDependencies()):
nodes[node.nodeID] = {
'hostname' : node.hostname,
'ipv6_addresses' : node.publicIPv6Addresses

View file

@ -1,84 +0,0 @@
from parser.JsonParser import JsonParser
import collections
import json
class Hopglass(JsonParser):
def __init__(self, filePath):
super().__init__(filePath)
self.ifIDs = {}
self.links = collections.defaultdict(dict)
self.nodes = {}
self.gatewayMacs = []
self.gateways = []
self.__aggregateData__()
# print(self.ifIDs)
# for k, v in self.links.items():
# print(k,v,'\n')
def __aggregateData__(self):
for nodeID, nodeData in self.__jsonData__.items():
# let pass nodes that provide all required informations only
if not set(('nodeinfo', 'neighbours', 'statistics')) <= set(nodeData):
continue
nodeInfo = nodeData['nodeinfo']
neighbours = nodeData['neighbours']
statistics = nodeData['statistics']
if not 'batadv' in neighbours:
continue
if not 'mesh' in nodeInfo.get('network', {}):
continue
if statistics.get('gateway', False):
self.gatewayMacs.append(statistics['gateway'])
for batID, batVal in nodeInfo['network']['mesh'].items():
if not 'interfaces' in batVal:
continue
for ifType, ifVal in batVal['interfaces'].items():
for mac in ifVal:
self.ifIDs[mac] = {
'type' : ifType,
'nodeID' : nodeID
}
self.nodes[nodeID] = nodeData
for nodeID, nodeData in self.nodes.items():
nodeData['nodeinfo']['isGateway'] = False
nodeData['nodeinfo']['isOnline'] = True # Todo: implement detection
for iname, ivalue in nodeData['neighbours']['batadv'].items():
if 'neighbours' not in ivalue:
continue
if iname in self.gatewayMacs:
nodeData['nodeinfo']['isGateway'] = True
if not iname in self.ifIDs:
continue
for nname, nvalue in ivalue['neighbours'].items():
if nname not in self.ifIDs:
continue
nifID = self.ifIDs[nname]['nodeID']
partID = (nodeID, nifID) if nodeID > nifID else (nifID, nodeID)
linkID = (iname, nname) if iname > nname else (nname, iname)
linkNode = {
'nodeID' : nodeID,
'type' : self.ifIDs[iname]['type'],
'tq' : nvalue['tq']
}
if linkID in self.links[partID]:
self.links[partID][linkID].append(linkNode)
else:
self.links[partID][linkID] = [linkNode]
def getLinksForNodeID(self, nodeID):
links = []
for link in self.links:
if link['target']['node_id'] == nodeID or link['source']['node_id'] == nodeID:
links.append(link)
return links

View file

@ -9,7 +9,7 @@ class JsonParser(object):
def __getFile__(self, fileName):
if fileName.startswith('https://') or fileName.startswith('http://'):
if self.printStatus:
print('Download', fileName, 'from URL:', fileName)
print('Download', fileName.rsplit('/', 1)[1] , 'from URL:', fileName)
resource = urllib.request.urlopen(fileName)
try:
data = json.loads(resource.read().decode('utf-8'))
@ -19,7 +19,7 @@ class JsonParser(object):
resource.close()
else:
if self.printStatus:
print('Open', fileName, 'from file:', fileName)
print('Open', fileName.rsplit('/', 1)[1] , 'from file:', fileName)
with open(fileName) as data_file:
try:
data = json.load(data_file)

View file

@ -7,12 +7,6 @@ class ShapesParser(JsonParser):
def __createShapes__(self):
shapes = []
if 'features' in self.__jsonData__:
for feature in self.__jsonData__['features']:
if feature['geometry']:
shapes.append(shape(feature['geometry']))
elif 'geometries' in self.__jsonData__:
for geometry in self.__jsonData__['geometries']:
shapes.append(shape(geometry))
for feature in self.__jsonData__['features']:
shapes.append(shape(feature['geometry']))
return shapes

View file

@ -1 +0,0 @@
shapely