freifunk/node_hierarchy
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Compare commits

base: freifunk:master
Branches Tags
freifunk:master
freifunk:hopglass
freifunk:meshviewer
freifunk:master_old_version_v1
..
compare: freifunk:meshviewer
Branches Tags
freifunk:master
freifunk:hopglass
freifunk:meshviewer
freifunk:master_old_version_v1

No commits in common. "master" and "meshviewer" have entirely different histories.
master ... meshviewer

13 changed files with 126 additions and 203 deletions
Show stats Expand all files Collapse all files

48
NodeHierarchy.py
View file

@ -1,8 +1,7 @@
#!/usr/bin/python3
import argparse
# from parser.NodesParser import NodesParser
# from parser.GraphParser import GraphParser
from parser.Hopglass import Hopglass
from parser.NodesParser import NodesParser
from parser.GraphParser import GraphParser
from cloud.Node import Node
from cloud.Link import Link
from cloud.GlobalGraph import GlobalGraph
@ -14,9 +13,8 @@ from info.Info import Info
class NodeHierarchy(object):
def __init__(self):
self.__args__ = self.__parseArguments__()
self.__hopglass = Hopglass(self.__args__.raw_json)
# self.__nodesJson__ = NodesParser(self.__args__.json_path)
# self.__graphJson__ = GraphParser(self.__args__.json_path)
self.__nodesJson__ = NodesParser(self.__args__.json_path)
self.__graphJson__ = GraphParser(self.__args__.json_path)
self.__shapesJson__ = self.__parseShapes__()
self.nodes = self.__createNodeObjects__()
self.links = self.__createLinkObjects__()
@ -42,29 +40,26 @@ class NodeHierarchy(object):
def __createNodeObjects__(self):
nodes = {}
for nodeID, nodeValue in self.__hopglass.nodes.items():
if nodeValue['nodeinfo']['node_id']:
print('Create Node object #',len(nodes), '\r',end = '')
nodes[nodeID] = Node(self.__prepareNodeData__(nodeValue))
for nodeID, nodeValue in self.__nodesJson__.nodes.items():
print('Create Node object #',len(nodes), '\r',end = '')
nodes[nodeID] = Node(nodeValue)
print('')
return nodes
def __prepareNodeData__(self, nodeValue):
if self.__args__.site_to_target_prefix:
pref = self.__args__.site_to_target_prefix.split(',')
try:
nodeValue['nodeinfo']['system']['site_code'] = nodeValue['nodeinfo']['system']['site_code'].replace(pref[0],pref[1])
except:
pass
return nodeValue
def __createLinkObjects__(self):
links = []
for linkParID, linkPar in self.__hopglass.links.items():
for linkID, link in linkPar.items():
print('Create Link object #',len(links), '\r',end = '')
if linkParID[0] != 'null' and linkParID[1] != 'null':
links.append(Link(link, (self.nodes[linkParID[0]], self.nodes[linkParID[1]])))
for link in self.__graphJson__.links:
try:
srcNode = self.nodes[link['source']['node_id']]
except:
srcNode = None
try:
dstNode = self.nodes[link['target']['node_id']]
except:
dstNode = None
print('Create Link object #',len(links), '\r',end = '')
links.append(Link(link, srcNode, dstNode))
print('')
return links
@ -73,13 +68,12 @@ class NodeHierarchy(object):
def __parseArguments__(self):
parser = argparse.ArgumentParser(description='This Script generates a hierarchical nodes list for node migration using nginx geo feature.')
parser.add_argument('-r', '--raw-json', required=False, default='https://karte.freifunk-muensterland.de/data/raw.json', help='Location of raw.json file (can be local folder or remote URL).')
parser.add_argument('-j', '--json-path', required=False, default='https://service.freifunk-muensterland.de/maps/data/', help='Path of nodes.json and graph.json (can be local folder or remote URL).')
parser.add_argument('-s', '--shapes-path', required=False, default='https://freifunk-muensterland.de/md-fw-dl/shapes/', help='Path of shapefiles (can be local folder or remote URL).')
parser.add_argument('-t', '--targets', nargs='+', required=True, help='List of targets which should be proceeded. Example: -t citya cityb ...')
parser.add_argument('-sttp', '--site-to-target-prefix', required=False, help='Used to match site and target also when prefixes are different. Example: -sttp "ffmsd,domaene"')
parser.add_argument('-o', '--out-file', default='./webserver-configuration', required=False, help='Filename where the generated Output should stored.')
parser.add_argument('-v', '--debug', required=False, action='store_true', help='Enable debugging output.')
parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan', 'domain_transitions_only'), help='Filter out nodes and local clouds based on filter rules.')
parser.add_argument('-f', '--filters', nargs='*', required=False, choices=('exclude_clouds_with_lan_links', 'no_lan'), help='Filter out nodes and local clouds based on filter rules.')
parser.add_argument('-i', '--info', nargs='*', required=False, choices=('get_offline_nodes','offline'), help='Get infos about the graph, links and nodes.')
parser.add_argument('-if', '--info-filters', nargs='*', required=False, help='Filter info results. Currently supported: min_age:TIME_RANGE, max_age:TIME_RANGE. Examples: -if min_age:1d max_age:2w')
parser.add_argument('-iop', '--info-out-path', required=False, default='./', help='Folder where info files should be written. Default: ./')

29
README.md
View file

@ -1,5 +1,5 @@
# Node Hierarchy
Dieses Tool generiert auf Basis einer ``raw.json`` des [hopglass-server](https://github.com/hopglass/hopglass-server) sowie (Multi-)Polygonen (im [geojson](http://geojson.org/) Format) der einzelnen Zieldomänen eine [nginx](http://nginx.org/) Konfigurationsdatei (auf Basis des [Geo-Moduls](http://nginx.org/en/docs/http/ngx_http_geo_module.html)), um Knoten in der richtigen Reihenfolge umzuziehen.
Dieses Tool generiert auf Basis einer ``graph.json`` und ``nodes.json`` des [Meshviewers](https://github.com/ffnord/meshviewer/) sowie (Multi-)Polygonen (im [geojson](http://geojson.org/) Format) der einzelnen Zieldomänen eine [nginx](http://nginx.org/) Konfigurationsdatei (auf Basis des [Geo-Moduls](http://nginx.org/en/docs/http/ngx_http_geo_module.html)), um Knoten in der richtigen Reihenfolge umzuziehen.
## Vorgehensweise
@ -28,10 +28,9 @@ Die Hilfe liefert folgendes:
```
$ ./NodeHierarchy.py --help
usage: NodeHierarchy.py [-h] [-r RAW_JSON] [-s SHAPES_PATH] -t TARGETS
[TARGETS ...] [-sttp SITE_TO_TARGET_PREFIX]
[-o OUT_FILE] [-v]
[-f [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} ...]]]
usage: NodeHierarchy.py [-h] [-j JSON_PATH] [-s SHAPES_PATH] -t TARGETS
[TARGETS ...] [-o OUT_FILE] [-v]
[-f [{exclude_clouds_with_lan_links,no_lan} [{exclude_clouds_with_lan_links,no_lan} ...]]]
[-i [{get_offline_nodes,offline} [{get_offline_nodes,offline} ...]]]
[-if [INFO_FILTERS [INFO_FILTERS ...]]]
[-iop INFO_OUT_PATH]
@ -42,22 +41,19 @@ geo feature.
optional arguments:
-h, --help show this help message and exit
-r RAW_JSON, --raw-json RAW_JSON
Location of raw.json file (can be local folder or
remote URL).
-j JSON_PATH, --json-path JSON_PATH
Path of nodes.json and graph.json (can be local folder
or remote URL).
-s SHAPES_PATH, --shapes-path SHAPES_PATH
Path of shapefiles (can be local folder or remote
URL).
-t TARGETS [TARGETS ...], --targets TARGETS [TARGETS ...]
List of targets which should be proceeded. Example: -t
citya cityb ...
-sttp SITE_TO_TARGET_PREFIX, --site-to-target-prefix SITE_TO_TARGET_PREFIX
Used to match site and target also when prefixes are
different. Example: -sttp "ffmsd,domaene"
-o OUT_FILE, --out-file OUT_FILE
Filename where the generated Output should stored.
-v, --debug Enable debugging output.
-f [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} ...]], --filters [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} [{exclude_clouds_with_lan_links,no_lan,domain_transitions_only} ...]]
-f [{exclude_clouds_with_lan_links,no_lan} [{exclude_clouds_with_lan_links,no_lan} ...]], --filters [{exclude_clouds_with_lan_links,no_lan} [{exclude_clouds_with_lan_links,no_lan} ...]]
Filter out nodes and local clouds based on filter
rules.
-i [{get_offline_nodes,offline} [{get_offline_nodes,offline} ...]], --info [{get_offline_nodes,offline} [{get_offline_nodes,offline} ...]]
@ -76,7 +72,7 @@ optional arguments:
### Anmerkungen
- ``--targets`` Gibt die Namen der Ziele (Zieldomänen) an. Der Geo-Schalter in der nginx-Konfiguration wird ebenfalls diesen Namen tragen.
- ``--raw-json`` Gibt den Ort der raw.json (hopglass-server) an. Default: ``https://karte.freifunk-muensterland.de/data/raw.json``
- ``--json-path`` Gibt das Daten-Verzeichnis eures Meshviewers an. Default: ``https://service.freifunk-muensterland.de/maps/data/``
- ``--shapes-path`` Verzeichnis an dem die Shapefiles der einzelnen Ziel-Domänen liegen. Default: ``https://freifunk-muensterland.de/md-fw-dl/shapes/``
- *Anmerkung:* Es werden Dateien in Abhängigkeit mit den Target-Namen im Verzeichnis erwartet.
- *Beispiel:* Bei ``-targets domaene01 domaene02`` werden die Dateien ``domaene01.geojson`` und ``domaene02.geojson`` erwartet.
@ -93,8 +89,7 @@ Weitere Filterungen lassen sich über das ``--filters`` Attribut aktivieren.
Folgende Filter sind derzeit implementiert (zukünftig folgen noch weitere):
- ``exclude_clouds_with_lan_links`` bzw. ``no_lan`` filtert alle lokalen Wolken aus, in denen sich mindestens ein Mesh-on-LAN Link befindet
- ``domain_transitions_only`` filtert alle Knoten aus, die sich bereits in der richtigen Domäne befinden / die Firmware der richtigen Domäne besitzen
- ``exclude_clouds_with_lan_links`` bzw. ``no_lan`` Filtert alle lokalen Wolken aus, in denen sich mindestens ein Mesh-on-LAN Link befindet
## Nginx Konfiguration
@ -135,9 +130,9 @@ schreibt in die Datei ``./offline_nodes.csv`` (default-Einstellung der Schalter
## Bekannte Probleme
Wenn es sich bei der Quell-Domäne um eine L2TP-Domäne handelt, läuft das Tool derzeit nur, wenn [alfred](https://github.com/ffnord/ffnord-alfred-announce) oder respondd auf allen Gateway-Servern läuft.
Wenn es sich bei der Quell-Domäne um eine L2TP-Domäne handelt, läuft das Tool derzeit nur, wenn [alfred](https://github.com/ffnord/ffnord-alfred-announce) auf allen Gateway-Servern läuft.
*Anmerkung:* Wenn in der ``graph.json`` mehrere Domänen vorhanden sind und dort teilweise L2TP-Domänen vorhanden sind (dieses aber nicht das Gebiet eurer Zieldomäne betrifft), kann das sehr negative Auswirkungen auf die Laufzeit haben (> 30 Sekunden).
*Anmerkung:* Wenn in der ``nodes.json`` und ``graph.json`` mehrere Domänen vorhanden sind und dort teilweise L2TP-Domänen vorhanden sind (dieses aber nicht das Gebiet eurer Zieldomäne betrifft), kann das sehr negative Auswirkungen auf die Laufzeit haben (> 30 Sekunden).
## Lizenz

9
cloud/GlobalGraph.py
View file

@ -26,19 +26,18 @@ class GlobalGraph(Graph):
def __getConnectedNodes__(self, nodeID, trace = []):
neighNodeIDs = self.getNeighbourNodeIDsForNodeID(nodeID)
trace_new = list(set(trace + neighNodeIDs))
trace_new = trace[:] + [x for x in neighNodeIDs if x not in trace]
for neighNodeID in neighNodeIDs:
if neighNodeID not in trace:
trace_new = list(set(trace_new + self.__getConnectedNodes__(neighNodeID, trace_new)))
trace_new = trace_new + [x for x in self.__getConnectedNodes__(neighNodeID, trace_new) if x not in trace_new]
return trace_new
def __createLocalCloudByNodesList__(self, nodesIDList):
nodes = {}
links = []
for nodeID in nodesIDList:
if nodeID:
nodes[nodeID] = self.__nodes__[nodeID]
links = list(set(links + self.getLinksByNodeID(nodeID)))
nodes[nodeID] = self.__nodes__[nodeID]
links = links + [x for x in self.getLinksByNodeID(nodeID) if x not in links]
return LocalGraph(nodes, links, self.__enableDebugPrinting__)
def __debugPrint__(self):

2
cloud/Graph.py
View file

@ -26,7 +26,7 @@ class Graph(object):
if link.isVpn == False:
endpoints = link.getEndpointNodeIDs(getGateways = False)
if nodeID in endpoints:
neighNodeIDs = list(set(neighNodeIDs + endpoints))
neighNodeIDs = neighNodeIDs + [x for x in endpoints if x not in neighNodeIDs]
return neighNodeIDs
def getLinksByNodeID(self, nodeID):

72
cloud/Link.py
View file

@ -1,29 +1,67 @@
class Link(object):
def __init__(self, LinkJsonObject, nodes):
self.__jsonObject = LinkJsonObject
self.linkType, self.isVpn = self.__getLinkType__()
self.__nodes = nodes
def __init__(self, LinkJsonObject, srcNode, dstNode):
self.__jsonObject__ = LinkJsonObject
self.__srcNode__ = srcNode
self.__dstNode__ = dstNode
self.linkType = self.__getLinkType__()
self.isVpn = self.__getLinkVpnState__()
def __getLinkType__(self):
types = [x['type'] for x in self.__jsonObject]
ltype = types[0]
lvpn = False
for x in types:
if x != 'unknown' and x != 'other':
if x == 'l2tp' or x == 'tunnel':
lvpn = True
val = x
return ltype, lvpn
type_src = None
type_dst = None
if self.__srcNode__ != None:
for k, v in self.__srcNode__.interfaces.items():
if self.__jsonObject__['source']['interface_mac'] in v:
type_src = k
if self.__dstNode__ != None:
for k, v in self.__dstNode__.interfaces.items():
if self.__jsonObject__['target']['interface_mac'] in v:
type_dst = k
if type_src == type_dst:
if type_src == None:
return 'unknown'
return type_src
else:
if type_src == None:
return type_dst
elif type_dst == None:
return type_src
else:
#print(self.__srcNode__.hostname, type_src, '<-->', self.__dstNode__.hostname, type_dst)
if type_src == 'wireless':
return type_dst
else:
return type_src
def __getLinkVpnState__(self):
if self.__jsonObject__['vpn'] == True:
return True
for node in self.getEndpointNodes(getGateways = True):
if node.isGateway == True:
return True
return False
def getEndpointNodes(self, getGateways = False):
return self.__nodes
nodes = []
if self.__srcNode__ != None:
if getGateways == True or self.__srcNode__.isGateway == False:
nodes.append(self.__srcNode__)
if self.__dstNode__ != None:
if getGateways == True or self.__dstNode__.isGateway == False:
nodes.append(self.__dstNode__)
return nodes
def getEndpointNodeIDs(self, getGateways = True):
return [x.nodeID for x in self.__nodes]
nodeIDs = []
for node in self.getEndpointNodes(getGateways):
nodeIDs.append(node.nodeID)
return nodeIDs
def isNodeIDinLink(self, nodeID):
for x in self.__nodes:
if nodeID == x.nodeID:
for endpoint in self.getEndpointNodes():
if endpoint.nodeID == nodeID:
return True
return False

7
cloud/LocalGraph.py
View file

@ -125,9 +125,6 @@ class LocalGraph(Graph):
print('BranchesThatExistsInCloud:', self.getBranchesThatExistsInCloud())
print('lan links in cloud:')
for link in self.getLanLinksInCloud():
hosts = link.getEndpointNodes()
if len(hosts) == 1:
print(' ', hosts.hostname, 'has unknown neighbour.')
else:
print(' ', hosts[0].hostname, '<--->', hosts[1].hostname)
if link.__srcNode__ != None and link.__dstNode__ != None:
print(' ', link.__srcNode__.hostname, '<--->', link.__dstNode__.hostname)
print('=====')

31
cloud/NodeInit.py
View file

@ -4,13 +4,13 @@ class NodeInit(object):
self.nodeID = self.__jsonObject__['nodeinfo']['node_id']
self.interfaces = self.__getInterfaces__()
self.hostname = self.__jsonObject__['nodeinfo']['hostname']
self.isGateway = self.__jsonObject__['nodeinfo']['isGateway']
self.isGateway = self.__jsonObject__['flags']['gateway']
self.geo = self.__getGeo__()
self.isAutoupdaterEnabled = self.__getAutoupdaterStatus__()
self.autoupdaterBranch = self.__getBranch__()
self.isOnline = self.__jsonObject__['nodeinfo']['isOnline']
self.isOnline = self.__jsonObject__['flags']['online']
self.publicIPv6Addresses = self.__getPublicAddresses__()
self.domName = self.__getSiteCode__()
self.domID = self.__getSiteCode__()
def __getInterfaces__(self):
try:
@ -19,34 +19,31 @@ class NodeInit(object):
return {}
def __getAutoupdaterStatus__(self):
try:
if 'autoupdater' in self.__jsonObject__['nodeinfo']['software']:
return self.__jsonObject__['nodeinfo']['software']['autoupdater']['enabled']
except:
else:
return False
def __getBranch__(self):
try:
if 'autoupdater' in self.__jsonObject__['nodeinfo']['software']:
return self.__jsonObject__['nodeinfo']['software']['autoupdater']['branch']
except:
else:
return None
def __getGeo__(self):
try:
return {
'lat' : self.__jsonObject__['nodeinfo']['location']['latitude'],
'lon' : self.__jsonObject__['nodeinfo']['location']['longitude']
}
except:
return None
geo = {}
if 'location' in self.__jsonObject__['nodeinfo'] and 'latitude' in self.__jsonObject__['nodeinfo']['location'] and 'longitude' in self.__jsonObject__['nodeinfo']['location']:
geo['lat'] = self.__jsonObject__['nodeinfo']['location']['latitude']
geo['lon'] = self.__jsonObject__['nodeinfo']['location']['longitude']
return geo
return None
def __getPublicAddresses__(self):
addresses = []
try:
if 'addresses' in self.__jsonObject__['nodeinfo']['network']:
for address in self.__jsonObject__['nodeinfo']['network']['addresses']:
if not address.startswith('fe80'):
addresses.append(address)
except:
pass
return addresses
def __getSiteCode__(self):

10
generator/Filter.py
View file

@ -3,7 +3,7 @@ class Filter(object):
self.__args__ = args
self.__filters__ = self.__getFilters()
def filterLocalGraphs(self, domain, localGraphs):
def filterLocalGraphs(self, localGraphs):
filteredGraphs = []
for localGraph in localGraphs:
if localGraph.isAutoupdaterEnabledOnAllNodes() == False:
@ -21,14 +21,8 @@ class Filter(object):
def __getFilters(self):
return [] if self.__args__.filters == None else self.__args__.filters
def filterNodes(self, domain, nodes):
def filterNodes(self, nodes):
filteredNodes = []
for node in nodes:
if 'domain_transitions_only' in self.__filters__:
try:
if domain.name == node.domName:
continue
except:
pass
filteredNodes.append(node)
return filteredNodes

4
generator/NginxConfGen.py
View file

@ -17,9 +17,9 @@ class NginxConfGen(object):
def __genDomain__(self, domain):
nodes = {}
for localGraph in self.__filter__.filterLocalGraphs(domain, domain.localGraphs):
for localGraph in self.__filter__.filterLocalGraphs(domain.localGraphs):
try:
for node in self.__filter__.filterNodes(domain, localGraph.getNodesWithNoDependencies()):
for node in self.__filter__.filterNodes(localGraph.getNodesWithNoDependencies()):
nodes[node.nodeID] = {
'hostname' : node.hostname,
'ipv6_addresses' : node.publicIPv6Addresses

84
parser/Hopglass.py
View file

@ -1,84 +0,0 @@
from parser.JsonParser import JsonParser
import collections
import json
class Hopglass(JsonParser):
def __init__(self, filePath):
super().__init__(filePath)
self.ifIDs = {}
self.links = collections.defaultdict(dict)
self.nodes = {}
self.gatewayMacs = []
self.gateways = []
self.__aggregateData__()
# print(self.ifIDs)
# for k, v in self.links.items():
# print(k,v,'\n')
def __aggregateData__(self):
for nodeID, nodeData in self.__jsonData__.items():
# let pass nodes that provide all required informations only
if not set(('nodeinfo', 'neighbours', 'statistics')) <= set(nodeData):
continue
nodeInfo = nodeData['nodeinfo']
neighbours = nodeData['neighbours']
statistics = nodeData['statistics']
if not 'batadv' in neighbours:
continue
if not 'mesh' in nodeInfo.get('network', {}):
continue
if statistics.get('gateway', False):
self.gatewayMacs.append(statistics['gateway'])
for batID, batVal in nodeInfo['network']['mesh'].items():
if not 'interfaces' in batVal:
continue
for ifType, ifVal in batVal['interfaces'].items():
for mac in ifVal:
self.ifIDs[mac] = {
'type' : ifType,
'nodeID' : nodeID
}
self.nodes[nodeID] = nodeData
for nodeID, nodeData in self.nodes.items():
nodeData['nodeinfo']['isGateway'] = False
nodeData['nodeinfo']['isOnline'] = True # Todo: implement detection
for iname, ivalue in nodeData['neighbours']['batadv'].items():
if 'neighbours' not in ivalue:
continue
if iname in self.gatewayMacs:
nodeData['nodeinfo']['isGateway'] = True
if not iname in self.ifIDs:
continue
for nname, nvalue in ivalue['neighbours'].items():
if nname not in self.ifIDs:
continue
nifID = self.ifIDs[nname]['nodeID']
partID = (nodeID, nifID) if nodeID > nifID else (nifID, nodeID)
linkID = (iname, nname) if iname > nname else (nname, iname)
linkNode = {
'nodeID' : nodeID,
'type' : self.ifIDs[iname]['type'],
'tq' : nvalue['tq']
}
if linkID in self.links[partID]:
self.links[partID][linkID].append(linkNode)
else:
self.links[partID][linkID] = [linkNode]
def getLinksForNodeID(self, nodeID):
links = []
for link in self.links:
if link['target']['node_id'] == nodeID or link['source']['node_id'] == nodeID:
links.append(link)
return links

4
parser/JsonParser.py
View file

@ -9,7 +9,7 @@ class JsonParser(object):
def __getFile__(self, fileName):
if fileName.startswith('https://') or fileName.startswith('http://'):
if self.printStatus:
print('Download', fileName, 'from URL:', fileName)
print('Download', fileName.rsplit('/', 1)[1] , 'from URL:', fileName)
resource = urllib.request.urlopen(fileName)
try:
data = json.loads(resource.read().decode('utf-8'))
@ -19,7 +19,7 @@ class JsonParser(object):
resource.close()
else:
if self.printStatus:
print('Open', fileName, 'from file:', fileName)
print('Open', fileName.rsplit('/', 1)[1] , 'from file:', fileName)
with open(fileName) as data_file:
try:
data = json.load(data_file)

10
parser/ShapesParser.py
View file

@ -7,12 +7,6 @@ class ShapesParser(JsonParser):
def __createShapes__(self):
shapes = []
if 'features' in self.__jsonData__:
for feature in self.__jsonData__['features']:
if feature['geometry']:
shapes.append(shape(feature['geometry']))
elif 'geometries' in self.__jsonData__:
for geometry in self.__jsonData__['geometries']:
shapes.append(shape(geometry))
for feature in self.__jsonData__['features']:
shapes.append(shape(feature['geometry']))
return shapes

1
requirements.txt
View file

@ -1 +0,0 @@
shapely