From 88b136f0a3ef2aef99f5e70373ced16257b845f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Tue, 22 Dec 2015 22:59:50 +0100 Subject: [PATCH 01/11] First version of migration script --- geocode.py | 39 +++++++++++++++++ graph.py | 107 ++++++++++++++++++++++++++++++++++++++++++++++ node.py | 104 ++++++++++++++++++++++++++++++++++++++++++++ node_hierarchy.py | 94 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 344 insertions(+) create mode 100644 geocode.py create mode 100644 graph.py create mode 100644 node.py create mode 100755 node_hierarchy.py diff --git a/geocode.py b/geocode.py new file mode 100644 index 0000000..908b9e6 --- /dev/null +++ b/geocode.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +# -*- coding: utf-8 - +#import time +from geopy.geocoders import Nominatim +from blitzdb import Document, FileBackend +class GeoAssign(Document): + pass +class Geocode: + def __init__(self, geocoderCache = True, printStatus = False): + self.printStatus = printStatus + self.geocoderCache = geocoderCache + if self.geocoderCache: + self.db = FileBackend('./geo-cache') + def getGeo(self, lon, lat): + if self.geocoderCache: + try: + nodeObj = self.db.get(GeoAssign,{'lat' : lat, 'lon' : lon}) + nodeObj['cached'] = True + return nodeObj + except GeoAssign.DoesNotExist: + pass + if self.printStatus: + print('lon: '+str(lon)+', lat: '+str(lat)+' not in cache - start lookup at Nominatim-API') + geolocator = Nominatim() + location = geolocator.reverse([lat, lon], timeout=20) + if 'address' in location.raw: + location = location.raw['address'] + nodeObj = GeoAssign({ + 'lat' : lat, + 'lon' : lon, + 'payload' : location + }) + self.db.save(nodeObj) + self.db.commit() + nodeObj['cached'] = False + return nodeObj + else: + # got no results (i.e. coordinates are incorrect) + return None \ No newline at end of file diff --git a/graph.py b/graph.py new file mode 100644 index 0000000..00fcb56 --- /dev/null +++ b/graph.py @@ -0,0 +1,107 @@ +#!/usr/bin/python +# -*- coding: utf-8 - +#Imports: +import urllib +import json +from pprint import pprint +from node import Node +from geocode import Geocode + +class Graph: + def __init__(self, nodesData, graphData): + self.coder = Geocode(geocoderCache = True, printStatus = True) + self.data = graphData + self.nodes = nodesData + self.nodes_list = {} + self.parseNodes() + self.parseLinks() + self.calculateStepsToVpn() + self.findMissingGeo() + + def parseNodes(self): + for k,v in self.nodes['nodes'].iteritems(): + lat, lon = self.getGeo(k) + node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder) + self.nodes_list[k] = node + + def parseLinks(self): + link_nodes = self.data['batadv']['nodes'] + for link in self.data['batadv']['links']: + if 'node_id' in link_nodes[link['source']].keys() and 'node_id' in link_nodes[link['target']].keys():#else it is a vpn link + self.setLinkBetween(link_nodes[link['source']]['node_id'], link_nodes[link['target']]['node_id']) + else: + self.setVpnLink(link['source'], link['target']) + + def setLinkBetween(self, src, dst, stateOnline = True, lastSeen = None): + if src and dst: + self.nodes_list[src].links[dst] = { + 'node' : self.nodes_list[dst], + 'state_online' : stateOnline, + 'last_seen' : lastSeen + } + self.nodes_list[dst].links[src] = { + 'node' : self.nodes_list[src], + 'state_online' : stateOnline, + 'last_seen' : lastSeen + } + + def setVpnLink(self, src, dst): + if 'node_id' not in self.data['batadv']['nodes'][src].keys(): + if self.data['batadv']['nodes'][dst]['node_id']: + self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].stepsToVpn = 0 + elif 'node_id' not in self.data['batadv']['nodes'][dst].keys(): + if self.data['batadv']['nodes'][src]['node_id']: + self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].stepsToVpn = 0 + + def calculateStepsToVpn(self): + for node in self.nodes_list.itervalues(): + node.calculateStepsToVpn() + + def findMissingGeo(self): + for node in self.nodes_list.itervalues(): + node.findMissingGeo() + + def getAllLevelXNodes(self, level, online = True): + zmap = {} + for k,v in self.nodes_list.iteritems(): + if v.isOnline or online == False: + if v.stepsToVpn == level: + zmap[k] = v + return zmap + + + def getHostname(self,node_id): + return self.nodes['nodes'][node_id]['nodeinfo']['hostname'] + + def getGeo(self, node_id): + if 'location' in self.nodes['nodes'][node_id]['nodeinfo'] and 'latitude' in self.nodes['nodes'][node_id]['nodeinfo']['location'] and 'longitude' in self.nodes['nodes'][node_id]['nodeinfo']['location']: + return self.nodes['nodes'][node_id]['nodeinfo']['location']['latitude'], self.nodes['nodes'][node_id]['nodeinfo']['location']['longitude'] + return None, None + + def getPublicAddress(self,node_id): + if node_id in self.nodes['nodes']: + if 'addresses' in self.nodes['nodes'][node_id]['nodeinfo']['network']: + for address in self.nodes['nodes'][node_id]['nodeinfo']['network']['addresses']: + if address.startswith('2a03'): + return address + return None + + def getOnlineState(self,node_id): + return self.nodes['nodes'][node_id]['flags']['online'] + + + def getNodeCloudsIn(self, region): + results = {} + for k,v in self.getAllLevelXNodes(0).iteritems(): + if v.geodata != None and v.isOnline == True: + if v.isInRegion(region): + results.update(v.getNodeCloud({})) + print "Result:",len(results), region + return results + + def maxDepth(self): + maxDepth = 0 + for v in self.nodes_list.itervalues(): + if v.stepsToVpn > maxDepth: + maxDepth = v.stepsToVpn + return maxDepth+1 \ No newline at end of file diff --git a/node.py b/node.py new file mode 100644 index 0000000..2fec681 --- /dev/null +++ b/node.py @@ -0,0 +1,104 @@ +#!/usr/bin/python +# -*- coding: utf-8 - +from geocode import Geocode + +class Node(object): + def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None): + self.coder = coder + if self.coder == None: + self.coder = Geocode(geocoderCache = True, printStatus = True) + self.links = {} + self.nodeid = nodeid + self.ipv6 = ipv6 + self.hostname = hostname + self.stepsToVpn = -1 + self.isOnline = isOnline + self.lastSeen = lastSeen + self._geo = None + self.geodata = None + if lat != None and lon != None: + self.geo = { + 'lat' : lat, + 'lon' : lon + } + + + def addLink(self,nodeid, node): + if not nodeid in self.links: + self.links[nodeid] = node + else: + print "link still exists" + + def calculateStepsToVpn(self, trace = []): + if self.stepsToVpn != 0:#self.stepsToVpn == -1 doesn't work, cause the shortest path could be the path to a former trace member + own_trace = trace[:]#clone - trace for preventing loops in pathfinding in graph + own_trace.append(self.nodeid) + lowest = -1 + current = -1 + for k,v in self.links.iteritems(): + if k not in own_trace: + current = v['node'].calculateStepsToVpn(own_trace) + if lowest == -1 or current < lowest: + lowest = current + if lowest > -1: + self.stepsToVpn = lowest+1 + return self.stepsToVpn + + def findMissingGeo(self, trace = []): + if self.geo == None: + own_trace = trace[:] + own_trace.append(self.nodeid) + geo = None + for k,v in self.links.iteritems(): + if k not in own_trace: + geo = v['node'].findMissingGeo(own_trace) + if geo != None: + self.geo = geo.copy() + break + return geo + else: + return self.geo + + def getNodeCloud(self, nodes = {}): + nodes[self.nodeid] = self + for k,v in self.links.iteritems(): + if k not in nodes: + nodes = v['node'].getNodeCloud(nodes) + return nodes + + def isInRegion(self, regions): + #AND and OR Conditions are possible + val = False + if self.geodata == None: + return False + + for region in regions: + val = False + for k,v in region.iteritems(): + if k in self.geodata and self.geodata[k] == v: + val = True + else: + val = False + if val: + return True + return val + + + @property + def geo(self): + return self._geo + + @geo.setter + def geo(self, value): + self._geo = value + self.__get_geodata__() + + def __get_geodata__(self): + if self.geo != None: + result = self.coder.getGeo(self.geo['lon'], self.geo['lat']) + if result: + self.geodata = result['payload'] + if result['cached'] == False: + time.sleep(1) + else: + self['geodata'] = None \ No newline at end of file diff --git a/node_hierarchy.py b/node_hierarchy.py new file mode 100755 index 0000000..edf0751 --- /dev/null +++ b/node_hierarchy.py @@ -0,0 +1,94 @@ +#!/usr/bin/python +# -*- coding: utf-8 - +#Imports: +import json +from graph import Graph + +class NodeHierarchy: + def __init__(self, nodesFile, graphFile, dataPath = './', printStatus = False, targets = None): + self.printStatus = printStatus + self.targets = targets + self.nodesData = self.__getFile__(nodesFile) + self.graphData = self.__getFile__(graphFile) + self.dataPath = dataPath + + self.graph = Graph(self.nodesData, self.graphData) + if self.targets == None: + self.writeConfigFiles(self.graph.nodes_list,"all") + else: + nodes = {} + for k,v in self.targets.iteritems(): + nodes = self.graph.getNodeCloudsIn(v) + self.writeConfigFiles(nodes,k) + nodes = {} + + def __getFile__(self, nodesFile): + if nodesFile.startswith('https://') or nodesFile.startswith('http://'): + if self.printStatus: + print "Download node.json from URL: " + nodesFile + resource = urllib.urlopen(nodesFile) + else: + if self.printStatus: + print "Open node.json file: " + nodesFile + resource = open(nodesFile) + data = json.loads(resource.read()) + resource.close() + return data + + def writeConfigFiles(self,nodes_level, name): + maxDepth = self.maxDepth(nodes_level) + for i in range(0,maxDepth): + content = 'geo $switch {\n\tdefault\t0;' + f = open(self.dataPath+'/'+name+'_node_level'+str(i),'w') + for node in nodes_level.itervalues(): + if node.stepsToVpn == i: + if node.ipv6 and node.hostname: + content += '\n\t'+node.ipv6+'\t1;\t #'+node.hostname + #else: + # print node.nodeid + content += '\n}' + f.write(content.encode('utf8')) + f.close() + + def maxDepth(self, nodes): + maxDepth = 0 + for v in nodes.itervalues(): + if v.stepsToVpn > maxDepth: + maxDepth = v.stepsToVpn + return maxDepth+1 + + +targets = { + 'muenster' : [ + {'city' : u'Münster'}, + {'county' : u'Münster'} + ], + 'kreis_warendorf' : [ + {'county' : u'Kreis Warendorf'} + ], + 'kreis_coesfeld' : [ + {'county' : u'Kreis Coesfeld'} + ], + 'kreis_steinfurt_west' : [ + {'town' : u'48565'}, + {'village' : u'Wettringen'}, + {'town' : u'Ochtrup'}, + {'village' : u'Metelen'}, + {'town' : u'Horstmar'}, + {'village' : u'Laer'}, + {'village' : u'Nordwalde'}, + {'village' : u'Altenberge'} + ], + 'kreis_steinfurt_ost' : [ + {'town' : u'Emsdetten'}, + {'town' : u'Neuenkirchen'}, + {'town' : u'Rheine'}, + {'town' : u'Greven'}, + {'village' : u'Ladbergen'}, + {'town' : u'Lengerich'}, + {'town' : u'Tecklenburg'}, + {'village' : u'Lienen'}, + ] +} + +ds = NodeHierarchy(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = './results/', targets = targets) \ No newline at end of file From 52efe6a7f0e79f1c07cfb87c48beecca77f59ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Wed, 23 Dec 2015 14:16:18 +0100 Subject: [PATCH 02/11] Added targets to script --- domain_selector.py | 57 ++++++++++++++++++++++++ node.py | 1 + node_hierarchy.py | 107 +++++++++++++++++++-------------------------- 3 files changed, 102 insertions(+), 63 deletions(-) create mode 100644 domain_selector.py diff --git a/domain_selector.py b/domain_selector.py new file mode 100644 index 0000000..9f995da --- /dev/null +++ b/domain_selector.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# -*- coding: utf-8 - +#Imports: +import json +from graph import Graph +class DomainSelector: + def __init__(self, nodesFile, graphFile, dataPath = './', printStatus = False, targets = None): + self.printStatus = printStatus + self.targets = targets + self.nodesData = self.__getFile__(nodesFile) + self.graphData = self.__getFile__(graphFile) + self.dataPath = dataPath + + self.graph = Graph(self.nodesData, self.graphData) + if self.targets == None: + self.writeConfigFiles(self.graph.nodes_list,"all") + else: + nodes = {} + for k,v in self.targets.iteritems(): + nodes = self.graph.getNodeCloudsIn(v) + self.writeConfigFiles(nodes,k) + nodes = {} + + def __getFile__(self, nodesFile): + if nodesFile.startswith('https://') or nodesFile.startswith('http://'): + if self.printStatus: + print "Download node.json from URL: " + nodesFile + resource = urllib.urlopen(nodesFile) + else: + if self.printStatus: + print "Open node.json file: " + nodesFile + resource = open(nodesFile) + data = json.loads(resource.read()) + resource.close() + return data + + def writeConfigFiles(self,nodes_level, name): + maxDepth = self.maxDepth(nodes_level) + for i in range(0,maxDepth): + content = 'geo $switch {\n\tdefault\t0;' + f = open(self.dataPath+'/'+name+'_node_level'+str(i),'w') + for node in nodes_level.itervalues(): + if node.stepsToVpn == i: + if node.ipv6 and node.hostname: + content += '\n\t'+node.ipv6+'\t1;\t #'+node.hostname + #else: + # print node.nodeid + content += '\n}' + f.write(content.encode('utf8')) + f.close() + + def maxDepth(self, nodes): + maxDepth = 0 + for v in nodes.itervalues(): + if v.stepsToVpn > maxDepth: + maxDepth = v.stepsToVpn + return maxDepth+1 \ No newline at end of file diff --git a/node.py b/node.py index 2fec681..486b937 100644 --- a/node.py +++ b/node.py @@ -1,6 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 - from geocode import Geocode +import time class Node(object): def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None): diff --git a/node_hierarchy.py b/node_hierarchy.py index edf0751..a4b9d27 100755 --- a/node_hierarchy.py +++ b/node_hierarchy.py @@ -2,72 +2,18 @@ # -*- coding: utf-8 - #Imports: import json -from graph import Graph - -class NodeHierarchy: - def __init__(self, nodesFile, graphFile, dataPath = './', printStatus = False, targets = None): - self.printStatus = printStatus - self.targets = targets - self.nodesData = self.__getFile__(nodesFile) - self.graphData = self.__getFile__(graphFile) - self.dataPath = dataPath - - self.graph = Graph(self.nodesData, self.graphData) - if self.targets == None: - self.writeConfigFiles(self.graph.nodes_list,"all") - else: - nodes = {} - for k,v in self.targets.iteritems(): - nodes = self.graph.getNodeCloudsIn(v) - self.writeConfigFiles(nodes,k) - nodes = {} - - def __getFile__(self, nodesFile): - if nodesFile.startswith('https://') or nodesFile.startswith('http://'): - if self.printStatus: - print "Download node.json from URL: " + nodesFile - resource = urllib.urlopen(nodesFile) - else: - if self.printStatus: - print "Open node.json file: " + nodesFile - resource = open(nodesFile) - data = json.loads(resource.read()) - resource.close() - return data - - def writeConfigFiles(self,nodes_level, name): - maxDepth = self.maxDepth(nodes_level) - for i in range(0,maxDepth): - content = 'geo $switch {\n\tdefault\t0;' - f = open(self.dataPath+'/'+name+'_node_level'+str(i),'w') - for node in nodes_level.itervalues(): - if node.stepsToVpn == i: - if node.ipv6 and node.hostname: - content += '\n\t'+node.ipv6+'\t1;\t #'+node.hostname - #else: - # print node.nodeid - content += '\n}' - f.write(content.encode('utf8')) - f.close() - - def maxDepth(self, nodes): - maxDepth = 0 - for v in nodes.itervalues(): - if v.stepsToVpn > maxDepth: - maxDepth = v.stepsToVpn - return maxDepth+1 - +from domain_selector import DomainSelector targets = { - 'muenster' : [ - {'city' : u'Münster'}, - {'county' : u'Münster'} - ], +# 'muenster' : [ +# {'city' : u'Münster'}, +# {'county' : u'Münster'}, +# ], 'kreis_warendorf' : [ - {'county' : u'Kreis Warendorf'} + {'county' : u'Kreis Warendorf'}, ], 'kreis_coesfeld' : [ - {'county' : u'Kreis Coesfeld'} + {'county' : u'Kreis Coesfeld'}, ], 'kreis_steinfurt_west' : [ {'town' : u'48565'}, @@ -77,7 +23,7 @@ targets = { {'town' : u'Horstmar'}, {'village' : u'Laer'}, {'village' : u'Nordwalde'}, - {'village' : u'Altenberge'} + {'village' : u'Altenberge'}, ], 'kreis_steinfurt_ost' : [ {'town' : u'Emsdetten'}, @@ -88,7 +34,42 @@ targets = { {'town' : u'Lengerich'}, {'town' : u'Tecklenburg'}, {'village' : u'Lienen'}, + ], + 'muenster_stadt' : [ + {'city_district' : u'Münster-Mitte'}, + {'city_district' : u'Münster-Nord'}, + {'city_district' : u'Münster-Ost'}, + {'suburb' : u'Berg Fidel'}, + {'suburb' : u'Gremmendorf'}, + {'suburb' : u'Mecklenbeck'}, + {'suburb' : u'Gievenbeck'}, + {'suburb' : u'Nienberge'}, + {'suburb' : u'Roxel'}, + {'suburb' : u'Sentruper Höhe'}, + ], + 'muenster_sued' : [ + {'suburb' : u'Amelsbüren'}, + {'suburb' : u'Hiltrup'}, + {'suburb' : u'Albachten'}, + ], + 'kreis_borken' : [ + {'town' : u'Ahaus'}, + {'town' : u'Bocholt'}, + {'town' : u'Borken'}, + {'town' : u'Gescher'}, + {'village' : u'Heek'}, + {'town' : u'Heiden'}, + {'town' : u'Isselburg'}, + {'village' : u'Legden'}, + {'town' : u'Raesfeld'}, + {'town' : u'Reken'}, + {'town' : u'Rhede'}, + {'village' : u'Schöppingen'}, + {'town' : u'Stadtlohn'}, + {'village' : u'Südlohn'}, + {'town' : u'Velen'}, + {'town' : u'Vreden'}, ] } -ds = NodeHierarchy(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = './results/', targets = targets) \ No newline at end of file +ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) \ No newline at end of file From 48d4fca08ba87e49f4dd55052d827d581e339e18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Wed, 23 Dec 2015 17:35:55 +0100 Subject: [PATCH 03/11] Implemented export of statistic files and first version of statistics file --- domain_selector.py | 21 +++++++++-- node_hierarchy.py | 1 - track_statistics.py | 90 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 4 deletions(-) create mode 100755 track_statistics.py diff --git a/domain_selector.py b/domain_selector.py index 9f995da..2925cad 100644 --- a/domain_selector.py +++ b/domain_selector.py @@ -14,11 +14,13 @@ class DomainSelector: self.graph = Graph(self.nodesData, self.graphData) if self.targets == None: self.writeConfigFiles(self.graph.nodes_list,"all") + self.writeDumpFile(self.graph.nodes_list,"all") else: nodes = {} for k,v in self.targets.iteritems(): nodes = self.graph.getNodeCloudsIn(v) self.writeConfigFiles(nodes,k) + self.writeDumpFile(nodes,k) nodes = {} def __getFile__(self, nodesFile): @@ -34,12 +36,12 @@ class DomainSelector: resource.close() return data - def writeConfigFiles(self,nodes_level, name): - maxDepth = self.maxDepth(nodes_level) + def writeConfigFiles(self, nodes, name): + maxDepth = self.maxDepth(nodes) for i in range(0,maxDepth): content = 'geo $switch {\n\tdefault\t0;' f = open(self.dataPath+'/'+name+'_node_level'+str(i),'w') - for node in nodes_level.itervalues(): + for node in nodes.itervalues(): if node.stepsToVpn == i: if node.ipv6 and node.hostname: content += '\n\t'+node.ipv6+'\t1;\t #'+node.hostname @@ -49,6 +51,19 @@ class DomainSelector: f.write(content.encode('utf8')) f.close() + def writeDumpFile(self, nodes, name): + content = {} + for node in nodes.itervalues(): + if node.ipv6 and node.hostname: + content[node.nodeid] = { + 'nodeid' : node.nodeid, + 'ipv6' : node.ipv6, + 'hostname' : node.hostname, + 'level' : node.stepsToVpn, + } + with open(self.dataPath+'/'+name+'_node_statistics.json', 'w') as outfile: + json.dump(content, outfile) + def maxDepth(self, nodes): maxDepth = 0 for v in nodes.itervalues(): diff --git a/node_hierarchy.py b/node_hierarchy.py index a4b9d27..7660ce4 100755 --- a/node_hierarchy.py +++ b/node_hierarchy.py @@ -1,7 +1,6 @@ #!/usr/bin/python # -*- coding: utf-8 - #Imports: -import json from domain_selector import DomainSelector targets = { diff --git a/track_statistics.py b/track_statistics.py new file mode 100755 index 0000000..33617c7 --- /dev/null +++ b/track_statistics.py @@ -0,0 +1,90 @@ +#!/usr/bin/python +# -*- coding: utf-8 - +#Imports: +import json +import operator + +class TrackStatistics: + def __init__(self,nodesFile,statisticsFiles, printStatus = False): + self.printStatus = printStatus + self.nodesData = self.__getFile__(nodesFile) + self.statisticsData = [] + for entry in statisticsFiles: + self.statisticsData.append({'data' : self.__getFile__(entry['data']), 'name' : entry['name']}) + #print self.statisticsData + for domain in self.statisticsData: + self.printDomainStatisticsPerLevel(domain['data'], domain['name']) + + def printDomainStatisticsPerLevel(self,data, name = "not set"): + #firmwareVersion = {} + print '-'*50 + print 'Printing statistics for domain:', name + for level in range(0,self.maxDepth(data)): + firmwareVersion = {} + for nodeid, node in data.iteritems(): + if level == node['level']: + fwver = self.nodesData['nodes'][nodeid]['nodeinfo']['software']['firmware']['release'] + if fwver in firmwareVersion: + firmwareVersion[fwver] += 1 + else: + firmwareVersion[fwver] = 1 + print '\tLevel:',level + for k,v in sorted(firmwareVersion.items(), key=operator.itemgetter(1), reverse = True): + print '\t\t '+k+':\t'+str(v) + print '-'*50 + + + + + def maxDepth(self, nodes): + maxDepth = 0 + for v in nodes.itervalues(): + if v['level'] > maxDepth: + maxDepth = v['level'] + return maxDepth+1 + + def __getFile__(self, nodesFile): + if nodesFile.startswith('https://') or nodesFile.startswith('http://'): + if self.printStatus: + print "Download node.json from URL: " + nodesFile + resource = urllib.urlopen(nodesFile) + else: + if self.printStatus: + print "Open node.json file: " + nodesFile + resource = open(nodesFile) + data = json.loads(resource.read()) + resource.close() + return data + +data = [ + { + 'data' : '../domaenensplit_webserver_config/muenster_sued_node_statistics.json', + 'name' : 'Münster Süd' + }, + { + 'data' : '../domaenensplit_webserver_config/muenster_stadt_node_statistics.json', + 'name' : 'Münster Stadt' + }, + { + 'data' : '../domaenensplit_webserver_config/kreis_coesfeld_node_statistics.json', + 'name' : 'Kreis Coesfeld' + }, + { + 'data' : '../domaenensplit_webserver_config/kreis_warendorf_node_statistics.json', + 'name' : 'Kreis Warendorf' + }, + { + 'data' : '../domaenensplit_webserver_config/kreis_steinfurt_ost_node_statistics.json', + 'name' : 'Kreis Steinfurt Ost' + }, + { + 'data' : '../domaenensplit_webserver_config/kreis_steinfurt_west_node_statistics.json', + 'name' : 'Kreis Steinfurt West' + }, + { + 'data' : '../domaenensplit_webserver_config/kreis_borken_node_statistics.json', + 'name' : 'Kreis Borken' + }, +] + +stat = TrackStatistics('nodes.json', data, printStatus = True) \ No newline at end of file From 1d66bb74181114de87d43a1c7567eab93e2eb8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Wed, 23 Dec 2015 22:51:35 +0100 Subject: [PATCH 04/11] Updated json file call from url --- domain_selector.py | 2 +- node_hierarchy.py | 3 ++- track_statistics.py | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/domain_selector.py b/domain_selector.py index 2925cad..c89fca3 100644 --- a/domain_selector.py +++ b/domain_selector.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 - #Imports: -import json +import json, urllib from graph import Graph class DomainSelector: def __init__(self, nodesFile, graphFile, dataPath = './', printStatus = False, targets = None): diff --git a/node_hierarchy.py b/node_hierarchy.py index 7660ce4..ac23bf9 100755 --- a/node_hierarchy.py +++ b/node_hierarchy.py @@ -71,4 +71,5 @@ targets = { ] } -ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) \ No newline at end of file +#ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) +ds = DomainSelector(nodesFile = 'https://freifunk-muensterland.de/map/data/nodes.json', graphFile = 'https://freifunk-muensterland.de/map/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) \ No newline at end of file diff --git a/track_statistics.py b/track_statistics.py index 33617c7..a818d54 100755 --- a/track_statistics.py +++ b/track_statistics.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 - #Imports: -import json +import json, urllib import operator class TrackStatistics: @@ -87,4 +87,5 @@ data = [ }, ] -stat = TrackStatistics('nodes.json', data, printStatus = True) \ No newline at end of file +#stat = TrackStatistics('nodes.json', data, printStatus = True) +stat = TrackStatistics('https://freifunk-muensterland.de/map/data/nodes.json', data, printStatus = True) \ No newline at end of file From 330e78c85f636fd7f08b2d1e13ca01e525eba9d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Fri, 8 Jan 2016 17:24:41 +0100 Subject: [PATCH 05/11] Added check for disabled autoupdater and branch --- domain_selector.py | 4 ++-- graph.py | 26 +++++++++++++++++++++++--- node.py | 4 +++- node_hierarchy.py | 11 ++++++++++- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/domain_selector.py b/domain_selector.py index c89fca3..1891e39 100644 --- a/domain_selector.py +++ b/domain_selector.py @@ -4,7 +4,7 @@ import json, urllib from graph import Graph class DomainSelector: - def __init__(self, nodesFile, graphFile, dataPath = './', printStatus = False, targets = None): + def __init__(self, nodesFile, graphFile, dataPath = './', printStatus = False, targets = None, branch = 'stable'): self.printStatus = printStatus self.targets = targets self.nodesData = self.__getFile__(nodesFile) @@ -18,7 +18,7 @@ class DomainSelector: else: nodes = {} for k,v in self.targets.iteritems(): - nodes = self.graph.getNodeCloudsIn(v) + nodes = self.graph.getNodeCloudsIn(v,branch) self.writeConfigFiles(nodes,k) self.writeDumpFile(nodes,k) nodes = {} diff --git a/graph.py b/graph.py index 00fcb56..99fcffc 100644 --- a/graph.py +++ b/graph.py @@ -21,7 +21,7 @@ class Graph: def parseNodes(self): for k,v in self.nodes['nodes'].iteritems(): lat, lon = self.getGeo(k) - node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder) + node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder, autoupdater = self.getAutoupdaterStatus(k), branch = self.getBranch(k)) self.nodes_list[k] = node def parseLinks(self): @@ -73,6 +73,22 @@ class Graph: def getHostname(self,node_id): return self.nodes['nodes'][node_id]['nodeinfo']['hostname'] + def getAutoupdaterStatus(self, node_id): + #return True + if 'autoupdater' in self.nodes['nodes'][node_id]['nodeinfo']['software']: + return self.nodes['nodes'][node_id]['nodeinfo']['software']['autoupdater']['enabled'] + else: + #if node is offline for a long time sometimes no autoupdater status can be found + return False + + def getBranch(self, node_id): + #return True + if 'autoupdater' in self.nodes['nodes'][node_id]['nodeinfo']['software']: + return self.nodes['nodes'][node_id]['nodeinfo']['software']['autoupdater']['branch'] + else: + #if node is offline for a long time sometimes no autoupdater status can be found + return None + def getGeo(self, node_id): if 'location' in self.nodes['nodes'][node_id]['nodeinfo'] and 'latitude' in self.nodes['nodes'][node_id]['nodeinfo']['location'] and 'longitude' in self.nodes['nodes'][node_id]['nodeinfo']['location']: return self.nodes['nodes'][node_id]['nodeinfo']['location']['latitude'], self.nodes['nodes'][node_id]['nodeinfo']['location']['longitude'] @@ -90,12 +106,16 @@ class Graph: return self.nodes['nodes'][node_id]['flags']['online'] - def getNodeCloudsIn(self, region): + def getNodeCloudsIn(self, region, branch = 'stable'): results = {} for k,v in self.getAllLevelXNodes(0).iteritems(): if v.geodata != None and v.isOnline == True: if v.isInRegion(region): - results.update(v.getNodeCloud({})) + for ksub,vsub in v.getNodeCloud({}).iteritems(): + if not vsub.autoupdater or vsub.branch != branch: + break + else: + results.update(v.getNodeCloud({})) print "Result:",len(results), region return results diff --git a/node.py b/node.py index 486b937..a67718b 100644 --- a/node.py +++ b/node.py @@ -4,7 +4,7 @@ from geocode import Geocode import time class Node(object): - def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None): + def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None, autoupdater = False, branch = None): self.coder = coder if self.coder == None: self.coder = Geocode(geocoderCache = True, printStatus = True) @@ -15,6 +15,8 @@ class Node(object): self.stepsToVpn = -1 self.isOnline = isOnline self.lastSeen = lastSeen + self.autoupdater = autoupdater + self.branch = branch self._geo = None self.geodata = None if lat != None and lon != None: diff --git a/node_hierarchy.py b/node_hierarchy.py index ac23bf9..abe1170 100755 --- a/node_hierarchy.py +++ b/node_hierarchy.py @@ -68,8 +68,17 @@ targets = { {'village' : u'Südlohn'}, {'town' : u'Velen'}, {'town' : u'Vreden'}, + ], + 'sassenberg' : [ + {'town' : u'Sassenberg'}, + ], + 'telgte' : [ + {'town' : u'Telgte'}, + ], + 'warendorf_stadt' : [ + {'town' : u'Warendorf'}, ] } #ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) -ds = DomainSelector(nodesFile = 'https://freifunk-muensterland.de/map/data/nodes.json', graphFile = 'https://freifunk-muensterland.de/map/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) \ No newline at end of file +ds = DomainSelector(nodesFile = 'https://freifunk-muensterland.de/map/data/nodes.json', graphFile = 'https://freifunk-muensterland.de/map/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = 'stable') \ No newline at end of file From bd1798195b7fcb86a009612bcb46ac78efa3323a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Sat, 26 Mar 2016 14:29:23 +0100 Subject: [PATCH 06/11] - Added support of GW-mesh-links (Domaene 14) - Added output of nodes without geo or disabled autoupdater - Changed output styling - Several minor changes --- domain_selector.py | 10 ++-- graph.py | 42 ++++++++----- no_coords.py | 116 ++++++++++++++++++++++++++++++++++++ node.py | 3 +- node_hierarchy.py | 143 ++++++++++++++++++++++++--------------------- 5 files changed, 230 insertions(+), 84 deletions(-) create mode 100755 no_coords.py diff --git a/domain_selector.py b/domain_selector.py index 1891e39..5b83fa6 100644 --- a/domain_selector.py +++ b/domain_selector.py @@ -22,6 +22,8 @@ class DomainSelector: self.writeConfigFiles(nodes,k) self.writeDumpFile(nodes,k) nodes = {} + self.writeConfigFiles(self.graph.nodes_no_autoupdater,"no_autoupdater") + self.writeConfigFiles(self.graph.nodes_no_geo,"no_geo") def __getFile__(self, nodesFile): if nodesFile.startswith('https://') or nodesFile.startswith('http://'): @@ -39,14 +41,12 @@ class DomainSelector: def writeConfigFiles(self, nodes, name): maxDepth = self.maxDepth(nodes) for i in range(0,maxDepth): - content = 'geo $switch {\n\tdefault\t0;' + content = 'geo $switch {\n default 0;' f = open(self.dataPath+'/'+name+'_node_level'+str(i),'w') for node in nodes.itervalues(): if node.stepsToVpn == i: if node.ipv6 and node.hostname: - content += '\n\t'+node.ipv6+'\t1;\t #'+node.hostname - #else: - # print node.nodeid + content += '\n '+node.ipv6+' 1; #'+node.hostname content += '\n}' f.write(content.encode('utf8')) f.close() @@ -69,4 +69,4 @@ class DomainSelector: for v in nodes.itervalues(): if v.stepsToVpn > maxDepth: maxDepth = v.stepsToVpn - return maxDepth+1 \ No newline at end of file + return maxDepth+1 diff --git a/graph.py b/graph.py index 99fcffc..bb090e9 100644 --- a/graph.py +++ b/graph.py @@ -13,6 +13,8 @@ class Graph: self.data = graphData self.nodes = nodesData self.nodes_list = {} + self.nodes_no_autoupdater = {} + self.nodes_no_geo = {} self.parseNodes() self.parseLinks() self.calculateStepsToVpn() @@ -21,14 +23,17 @@ class Graph: def parseNodes(self): for k,v in self.nodes['nodes'].iteritems(): lat, lon = self.getGeo(k) - node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder, autoupdater = self.getAutoupdaterStatus(k), branch = self.getBranch(k)) + node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder, autoupdater = self.getAutoupdaterStatus(k), branch = self.getBranch(k), isGateway = self.getIsGateway(k)) self.nodes_list[k] = node def parseLinks(self): link_nodes = self.data['batadv']['nodes'] for link in self.data['batadv']['links']: if 'node_id' in link_nodes[link['source']].keys() and 'node_id' in link_nodes[link['target']].keys():#else it is a vpn link - self.setLinkBetween(link_nodes[link['source']]['node_id'], link_nodes[link['target']]['node_id']) + if self.nodes_list[link_nodes[link['source']]['node_id']].isGateway == True or self.nodes_list[link_nodes[link['target']]['node_id']].isGateway: + self.setVpnLink(link['source'], link['target']) + else: + self.setLinkBetween(link_nodes[link['source']]['node_id'], link_nodes[link['target']]['node_id']) else: self.setVpnLink(link['source'], link['target']) @@ -46,11 +51,11 @@ class Graph: } def setVpnLink(self, src, dst): - if 'node_id' not in self.data['batadv']['nodes'][src].keys(): - if self.data['batadv']['nodes'][dst]['node_id']: + if 'node_id' not in self.data['batadv']['nodes'][src].keys() or self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].isGateway == True: + if 'node_id' in self.data['batadv']['nodes'][dst]: self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].stepsToVpn = 0 - elif 'node_id' not in self.data['batadv']['nodes'][dst].keys(): - if self.data['batadv']['nodes'][src]['node_id']: + elif 'node_id' not in self.data['batadv']['nodes'][dst].keys() or self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].isGateway == True: + if 'node_id' in self.data['batadv']['nodes'][src]: self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].stepsToVpn = 0 def calculateStepsToVpn(self): @@ -73,6 +78,9 @@ class Graph: def getHostname(self,node_id): return self.nodes['nodes'][node_id]['nodeinfo']['hostname'] + def getIsGateway(self,node_id): + return self.nodes['nodes'][node_id]['flags']['gateway'] + def getAutoupdaterStatus(self, node_id): #return True if 'autoupdater' in self.nodes['nodes'][node_id]['nodeinfo']['software']: @@ -108,14 +116,22 @@ class Graph: def getNodeCloudsIn(self, region, branch = 'stable'): results = {} + noAuto = False for k,v in self.getAllLevelXNodes(0).iteritems(): - if v.geodata != None and v.isOnline == True: - if v.isInRegion(region): - for ksub,vsub in v.getNodeCloud({}).iteritems(): - if not vsub.autoupdater or vsub.branch != branch: - break - else: - results.update(v.getNodeCloud({})) + if v.isOnline == True: + if v.geodata != None: + if v.isInRegion(region): + noAuto = False + for ksub,vsub in v.getNodeCloud({}).iteritems(): + if not vsub.autoupdater or (branch and vsub.branch != branch): + #break + noAuto = True + self.nodes_no_autoupdater[ksub] = vsub + #else: + if not noAuto: + results.update(v.getNodeCloud({})) + else: + self.nodes_no_geo.update(v.getNodeCloud({})) print "Result:",len(results), region return results diff --git a/no_coords.py b/no_coords.py new file mode 100755 index 0000000..3b9093e --- /dev/null +++ b/no_coords.py @@ -0,0 +1,116 @@ +#!/usr/bin/python +# +# (c) 2016 descilla +# +# This script is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License or any later version. +# +# This script is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY. See the +# GNU General Public License for more details. +# +# For a copy of the GNU General Public License +# see . +# +import glob, os, json, collections, argparse, urllib, datetime +from collections import OrderedDict + +class OfflineChecker: + def __init__(self, dataFile): + self.printStatus = False + self.dataSet = self.__getFile__(dataFile) + self.fileNames = [] + self.results = {} + self.addresses = [] + self.getAddressFiles() + self.getFwState() + + def __getFile__(self, nodesFile): + if nodesFile.startswith('https://') or nodesFile.startswith('http://'): + if self.printStatus: + print "Download node.json from URL: " + nodesFile + resource = urllib.urlopen(nodesFile) + else: + if self.printStatus: + print "Open node.json file: " + nodesFile + resource = open(nodesFile) + data = json.loads(resource.read()) + resource.close() + return data + + def getAddressFiles(self): + for file in glob.iglob('./nodes_adresses_*'): + self.addresses.append(self.__getFile__(file)) + + def readFile(self): + fil = 'operate.txt' + results = [] + with open(fil) as lg: + for line in lg: + results.append(line) + return results + + def getNodeAddressItem(self,ipv6): + pub = "" + i = 0 + for adr in self.addresses: + i +=1 + for val in adr['nodes'].itervalues(): + pub = self.getPublicAddress(val) + if pub and pub in ipv6: + if 'owner' in val['nodeinfo']: + return val + + def getNodeItem(self,ipv6): + pub = "" + for val in self.dataSet['nodes'].itervalues(): + pub = self.getPublicAddress(val) + if pub and pub in ipv6: + return val + + def getFwState(self): + lastDay = datetime.datetime.today() - datetime.timedelta(hours = 48) + onlyoldernodes = False + results = {} + for nodeIP in self.readFile(): + nodeAddresses = self.getNodeAddressItem(nodeIP) + node = self.getNodeItem(nodeIP) + if node: + nodeLastSeen = datetime.datetime.strptime(node['lastseen'],'%Y-%m-%dT%H:%M:%S') + if nodeLastSeen < lastDay or onlyoldernodes == False: + au = node['nodeinfo']['software']['autoupdater']['branch'] + loca = 'JA' if 'location' in node['nodeinfo'] else 'NEIN' + if nodeAddresses: + mail = nodeAddresses['nodeinfo']['owner']['contact'] if 'owner' in nodeAddresses['nodeinfo'] else 'NEIN' + #mail = 'JA' if 'owner' in nodeAddresses['nodeinfo'] else 'NEIN' + else: + mail = 'NEIN' + results[node['nodeinfo']['node_id']] = { + 'lastseen' : node['lastseen'], + 'ipv6' : self.getPublicAddress(node), + 'node_id' : node['nodeinfo']['node_id'], + 'name' : node['nodeinfo']['hostname'], + 'contact' : mail, + 'fw_base' : node['nodeinfo']['software']['firmware']['base'], + 'fw_release' : node['nodeinfo']['software']['firmware']['release'], + 'au_enabled' : str(node['nodeinfo']['software']['autoupdater']['enabled']), + 'au_branch' : au, + 'router_modell' : node['nodeinfo']['hardware']['model'], + 'geo' : loca, + } + #print node['lastseen'] + ';' + self.getPublicAddress(node) + ';' + node['nodeinfo']['node_id'] + ';' + node['nodeinfo']['hostname'] + ';' + mail + ';' + node['nodeinfo']['software']['firmware']['base'] + ';' + node['nodeinfo']['software']['firmware']['release'] + ';' + str(node['nodeinfo']['software']['autoupdater']['enabled']) + ';' + au + ';' + node['nodeinfo']['hardware']['model'] + ';' + loca + self.printCSV(results) + def printCSV(self, data): + od = OrderedDict(sorted(data.items(), key=lambda x: x[1]['lastseen'], reverse=True)) + print 'zuletzt online;nodeid;Knotenname;mailaddress;Firmware Base;Firmware Release;Autoupdater;AU-Branch;Router-Modell;geo' + for item in od.itervalues(): + print item['lastseen'] + ';' + item['node_id'] + ';' + item['name'] + ';' + item['contact'] + ';' + item['fw_base'] + ';' + item['fw_release'] + ';' + item['au_enabled'] + ';' + item['au_branch'] + ';' + item['router_modell'] + ';' + item['geo'] + def getPublicAddress(self,node): + if 'addresses' in node['nodeinfo']['network']: + for address in node['nodeinfo']['network']['addresses']: + if address.startswith('2a03'): + return address + return None + +dmax = OfflineChecker('https://service.freifunk-muensterland.de/maps/data/nodes.json') \ No newline at end of file diff --git a/node.py b/node.py index a67718b..632de18 100644 --- a/node.py +++ b/node.py @@ -4,7 +4,7 @@ from geocode import Geocode import time class Node(object): - def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None, autoupdater = False, branch = None): + def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None, autoupdater = False, branch = None, isGateway = False): self.coder = coder if self.coder == None: self.coder = Geocode(geocoderCache = True, printStatus = True) @@ -19,6 +19,7 @@ class Node(object): self.branch = branch self._geo = None self.geodata = None + self.isGateway = isGateway if lat != None and lon != None: self.geo = { 'lat' : lat, diff --git a/node_hierarchy.py b/node_hierarchy.py index abe1170..1c4decb 100755 --- a/node_hierarchy.py +++ b/node_hierarchy.py @@ -8,77 +8,90 @@ targets = { # {'city' : u'Münster'}, # {'county' : u'Münster'}, # ], - 'kreis_warendorf' : [ - {'county' : u'Kreis Warendorf'}, - ], - 'kreis_coesfeld' : [ - {'county' : u'Kreis Coesfeld'}, - ], - 'kreis_steinfurt_west' : [ - {'town' : u'48565'}, - {'village' : u'Wettringen'}, - {'town' : u'Ochtrup'}, - {'village' : u'Metelen'}, - {'town' : u'Horstmar'}, - {'village' : u'Laer'}, - {'village' : u'Nordwalde'}, - {'village' : u'Altenberge'}, - ], - 'kreis_steinfurt_ost' : [ - {'town' : u'Emsdetten'}, - {'town' : u'Neuenkirchen'}, - {'town' : u'Rheine'}, - {'town' : u'Greven'}, - {'village' : u'Ladbergen'}, - {'town' : u'Lengerich'}, - {'town' : u'Tecklenburg'}, - {'village' : u'Lienen'}, - ], - 'muenster_stadt' : [ - {'city_district' : u'Münster-Mitte'}, - {'city_district' : u'Münster-Nord'}, - {'city_district' : u'Münster-Ost'}, - {'suburb' : u'Berg Fidel'}, - {'suburb' : u'Gremmendorf'}, - {'suburb' : u'Mecklenbeck'}, - {'suburb' : u'Gievenbeck'}, - {'suburb' : u'Nienberge'}, - {'suburb' : u'Roxel'}, - {'suburb' : u'Sentruper Höhe'}, - ], - 'muenster_sued' : [ - {'suburb' : u'Amelsbüren'}, - {'suburb' : u'Hiltrup'}, - {'suburb' : u'Albachten'}, - ], - 'kreis_borken' : [ - {'town' : u'Ahaus'}, - {'town' : u'Bocholt'}, - {'town' : u'Borken'}, - {'town' : u'Gescher'}, - {'village' : u'Heek'}, - {'town' : u'Heiden'}, - {'town' : u'Isselburg'}, - {'village' : u'Legden'}, - {'town' : u'Raesfeld'}, - {'town' : u'Reken'}, - {'town' : u'Rhede'}, - {'village' : u'Schöppingen'}, + # 'kreis_warendorf' : [ + # {'county' : u'Kreis Warendorf'}, + # ], + # 'kreis_coesfeld' : [ + # {'county' : u'Kreis Coesfeld'}, + # ], + # 'kreis_steinfurt_west' : [ + # {'town' : u'48565'}, + # {'village' : u'Wettringen'}, + # {'town' : u'Ochtrup'}, + # {'village' : u'Metelen'}, + # {'town' : u'Horstmar'}, + # {'village' : u'Laer'}, + # {'village' : u'Nordwalde'}, + # {'village' : u'Altenberge'}, + # ], + # 'kreis_steinfurt_ost' : [ + # {'town' : u'Emsdetten'}, + # {'town' : u'Neuenkirchen'}, + # {'town' : u'Rheine'}, + # {'town' : u'Greven'}, + # {'village' : u'Ladbergen'}, + # {'town' : u'Lengerich'}, + # {'town' : u'Tecklenburg'}, + # {'village' : u'Lienen'}, + # ], + # 'muenster_stadt' : [ + # {'city_district' : u'Münster-Mitte'}, + # {'city_district' : u'Münster-Nord'}, + # {'city_district' : u'Münster-Ost'}, + # {'suburb' : u'Berg Fidel'}, + # {'suburb' : u'Gremmendorf'}, + # {'suburb' : u'Mecklenbeck'}, + # {'suburb' : u'Gievenbeck'}, + # {'suburb' : u'Nienberge'}, + # {'suburb' : u'Roxel'}, + # {'suburb' : u'Sentruper Höhe'}, + # ], + # 'muenster_sued' : [ + # {'suburb' : u'Amelsbüren'}, + # {'suburb' : u'Hiltrup'}, + # {'suburb' : u'Albachten'}, + # ], + # 'kreis_borken' : [ + # {'town' : u'Ahaus'}, + # {'town' : u'Bocholt'}, + # {'town' : u'Borken'}, + # {'town' : u'Gescher'}, + # {'village' : u'Heek'}, + # {'town' : u'Heiden'}, + # {'town' : u'Isselburg'}, + # {'village' : u'Legden'}, + # {'town' : u'Raesfeld'}, + # {'town' : u'Reken'}, + # {'town' : u'Rhede'}, + # {'village' : u'Schöppingen'}, + # {'town' : u'Stadtlohn'}, + # {'village' : u'Südlohn'}, + # {'town' : u'Velen'}, + # {'town' : u'Vreden'}, + # ], + # 'sassenberg' : [ + # {'town' : u'Sassenberg'}, + # ], + # 'telgte' : [ + # {'town' : u'Telgte'}, + # ], + # 'warendorf_stadt' : [ + # {'town' : u'Warendorf'}, + # ] + 'stadt_stadtlohn' : [ {'town' : u'Stadtlohn'}, - {'village' : u'Südlohn'}, - {'town' : u'Velen'}, - {'town' : u'Vreden'}, ], - 'sassenberg' : [ - {'town' : u'Sassenberg'}, + 'stadt_bocholt' : [ + {'town' : u'Bocholt'}, ], - 'telgte' : [ + 'stadt_telgte' : [ {'town' : u'Telgte'}, ], - 'warendorf_stadt' : [ + 'stadt_warendorf' : [ {'town' : u'Warendorf'}, - ] + ], } #ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) -ds = DomainSelector(nodesFile = 'https://freifunk-muensterland.de/map/data/nodes.json', graphFile = 'https://freifunk-muensterland.de/map/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = 'stable') \ No newline at end of file +#ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None) +ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None) From e051d64780ff424c73dabad33d40e870bb4c3168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Sun, 27 Mar 2016 19:15:16 +0200 Subject: [PATCH 07/11] Handle with IDs, which are not appear in nodes list --- graph.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/graph.py b/graph.py index bb090e9..39dad55 100644 --- a/graph.py +++ b/graph.py @@ -29,7 +29,7 @@ class Graph: def parseLinks(self): link_nodes = self.data['batadv']['nodes'] for link in self.data['batadv']['links']: - if 'node_id' in link_nodes[link['source']].keys() and 'node_id' in link_nodes[link['target']].keys():#else it is a vpn link + if 'node_id' in link_nodes[link['source']] and 'node_id' in link_nodes[link['target']]:#else it is a vpn link if self.nodes_list[link_nodes[link['source']]['node_id']].isGateway == True or self.nodes_list[link_nodes[link['target']]['node_id']].isGateway: self.setVpnLink(link['source'], link['target']) else: @@ -51,12 +51,12 @@ class Graph: } def setVpnLink(self, src, dst): - if 'node_id' not in self.data['batadv']['nodes'][src].keys() or self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].isGateway == True: - if 'node_id' in self.data['batadv']['nodes'][dst]: - self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].stepsToVpn = 0 - elif 'node_id' not in self.data['batadv']['nodes'][dst].keys() or self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].isGateway == True: - if 'node_id' in self.data['batadv']['nodes'][src]: - self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].stepsToVpn = 0 + if 'node_id' not in self.data['batadv']['nodes'][src] or (self.data['batadv']['nodes'][src]['node_id'] and self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].isGateway == True): + if 'node_id' in self.data['batadv']['nodes'][dst] and self.data['batadv']['nodes'][dst]['node_id']: + self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].stepsToVpn = 0 + elif 'node_id' not in self.data['batadv']['nodes'][dst] or (self.data['batadv']['nodes'][dst]['node_id'] and self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].isGateway == True): + if 'node_id' in self.data['batadv']['nodes'][src] and self.data['batadv']['nodes'][src]['node_id']: + self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].stepsToVpn = 0 def calculateStepsToVpn(self): for node in self.nodes_list.itervalues(): From c6a167dde5533f4cb8f35fa2b3cc888df63bc1fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Tue, 29 Mar 2016 00:30:02 +0200 Subject: [PATCH 08/11] Updated node_hierarchy configuration - Added json file for target configuration - Added arguments handling for commandline - Added some error handlings - Several other minor changes --- .gitignore | 4 ++ README.md | 46 ++++++++++++++++ domain_selector.py | 24 ++++++-- get_state.py | 100 ++++++++++++++++++++++++++++++++++ hieraException.py | 7 +++ node_hierarchy.py | 133 +++++++++++++++------------------------------ targets.json | 75 +++++++++++++++++++++++++ 7 files changed, 294 insertions(+), 95 deletions(-) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 get_state.py create mode 100644 hieraException.py create mode 100644 targets.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3423407 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +nodes_* +*.pyc +geo-cache/ +contact/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..053f1c6 --- /dev/null +++ b/README.md @@ -0,0 +1,46 @@ +# Knoten Migrationstool +Dieses Tool dient zur Generierung von nginx-Konfigurationsdateien für die Migration von Knoten von einer Domäne in eine andere Domäne. Dabei wird berücksichtigt, welche Knoten von anderen Knoten abhängen um ins Internet zu kommen. + +## Konfiguration +In der ``targets``-Datei werden Domänen (oder ähnliche Ziele) definiert. Hierbei handelt es sich um eine JSON-Datei. Dabei werden dort Gebietsrelationen für den Nominatik-Geocoder eingetragen. + +## Aufruf +Es muss eine Datei mit den ``targets`` existieren. + +Im einfachsten Fall wird das Programm dann wie folgt aufgerufen: +``` +./node_hierarcy.py --all +``` + +Sollen spezielle ``nodes.json`` und ``graph.json`` Dateien verwendet werden, so kann der Datenpfad wie folgt angegeben werden (dabei kann es sich um einen lokalen Dateipfad als auch um eine http oder https URL handel): + +``` +./node_hierarcy.py --all --json-path https://service.freifunk-muensterland.de/maps/data/ +``` + +Eine Ausgabe des Fortschritts erhält man mit dem Schalter ``-p`` oder ``--print-status``: + +``` +./node_hierarcy.py --all -p +``` + +Eine Limitierung auf eine Auswahl an Targets aus der Targets-Datei kann mit dem Schalter ``-t`` oder ``--targets`` vorgenommen werden: +``` +./node_hierarcy.py -t domaene_01 domaene_02 --print-status +``` + +Weitere Hilfestellungen erhält mann mit ``-h`` oder ``--help``: +``` +./node_hierarcy.py +``` + +### Ein- und Ausgabe +Standardmäßig wird eine Datei ``targets.json`` erwartet. Soll diese Datei von einer anderen Stelle aufgerufen werden kann das ``--targets-file``-Argument verwendet werden: +``` +./node_hierarcy.py --targets-file /root/targets.json +``` + +Standardmäßig erfolgt die Ausgabe der generierten nginx-Konfigurationsdateien in das Verzeichnis ``./webserver-configuration/``. Das kann mit dem Schalter ``--out-path`` geändert werden: +``` +./node_hierarcy.py --out-path /root/config/ +``` \ No newline at end of file diff --git a/domain_selector.py b/domain_selector.py index 5b83fa6..76c0f55 100644 --- a/domain_selector.py +++ b/domain_selector.py @@ -1,10 +1,16 @@ #!/usr/bin/python # -*- coding: utf-8 - #Imports: -import json, urllib +import json, urllib, os from graph import Graph +from hieraException import HieraException + class DomainSelector: def __init__(self, nodesFile, graphFile, dataPath = './', printStatus = False, targets = None, branch = 'stable'): + if not os.path.isdir(dataPath): + print "\033[91mError:\033[0m Output folder was not found or is not writable. Given path:", dataPath + raise HieraException + self.printStatus = printStatus self.targets = targets self.nodesData = self.__getFile__(nodesFile) @@ -28,21 +34,27 @@ class DomainSelector: def __getFile__(self, nodesFile): if nodesFile.startswith('https://') or nodesFile.startswith('http://'): if self.printStatus: - print "Download node.json from URL: " + nodesFile + print 'Download', nodesFile.rsplit('/', 1)[1] , 'from URL:', nodesFile resource = urllib.urlopen(nodesFile) else: if self.printStatus: - print "Open node.json file: " + nodesFile + print 'Open', nodesFile.rsplit('/', 1)[1] , 'from file:', nodesFile resource = open(nodesFile) - data = json.loads(resource.read()) - resource.close() + try: + data = json.loads(resource.read()) + except: + print "\033[91mError:\033[0m Error while parsing a json file (perhapes misformed file): ", nodesFile + raise HieraException + finally: + resource.close() + return data def writeConfigFiles(self, nodes, name): maxDepth = self.maxDepth(nodes) for i in range(0,maxDepth): content = 'geo $switch {\n default 0;' - f = open(self.dataPath+'/'+name+'_node_level'+str(i),'w') + f = open(self.dataPath.rstrip('/')+'/'+name+'_node_level'+str(i),'w') for node in nodes.itervalues(): if node.stepsToVpn == i: if node.ipv6 and node.hostname: diff --git a/get_state.py b/get_state.py new file mode 100644 index 0000000..c8b53e5 --- /dev/null +++ b/get_state.py @@ -0,0 +1,100 @@ +#!/usr/bin/python +# +# (c) 2016 descilla +# +# This script is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License or any later version. +# +# This script is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY. See the +# GNU General Public License for more details. +# +# For a copy of the GNU General Public License +# see . +# +import glob, os, json, collections, argparse, urllib + +class OfflineChecker: + def __init__(self, fileName): + self.printStatus = True + self.fileNames = [] + self.results = {} + self.data = self.__getFile__(fileName) + self.addresses = self.__getFile__('nodes_legacy_adresses.json') + self.addressesOld = self.__getFile__('nodes_legacy_adresses_old.json') + self.parseJson(self.data) + self.getFwState() + #self.printResults() + + def __getFile__(self, nodesFile): + if nodesFile.startswith('https://') or nodesFile.startswith('http://'): + if self.printStatus: + print "Download node.json from URL: " + nodesFile + resource = urllib.urlopen(nodesFile) + else: + if self.printStatus: + print "Open node.json file: " + nodesFile + resource = open(nodesFile) + data = json.loads(resource.read()) + resource.close() + return data + + def searchInLog(self, key, arg): + files = ['/var/log/nginx/access.log', '/var/log/nginx/access.log.1'] + for fil in files: + with open(fil) as lg: + for line in lg: + if key and key in line: + if arg in line: + date = line.split('[')[1].split(']')[0] + dest_dom = line.split('gluon-')[1].split('-')[0] + return date, dest_dom + return None, None + + def parseJson(self, data): + nodes_online = 0 + users_online = 0 + day_stamp = data['timestamp'].split('T')[0] + for key, node in data['nodes'].iteritems(): + if 'statistics' in node: + users_online += node['statistics']['clients'] + if 'flags' in node: + if node['flags']['online'] == False: + if 'system' in node['nodeinfo']: + siteCode = node['nodeinfo']['system']['site_code'] + if siteCode not in self.results: + self.results[siteCode] = {} + self.results[siteCode][key] = { + 'lastseen' : node['lastseen'], + 'id' : key, + 'mac' : node['nodeinfo']['network']['mac'], + 'pub_v6' : self.getPublicAddress(node), + 'name' : node['nodeinfo']['hostname'] + } + + def getFwState(self): + print 'fw_geladen\tlastseen\tziel_dom\tipv6_adresse\tnodeid\thostname\tmailaddress' + for node, val in self.results['ffms'].iteritems(): + date, dest_dom = self.searchInLog(val['pub_v6'], "sysupgrade.bin") + if date and dest_dom: + #mail = self.addresses['nodes'][node]['nodeinfo']['owner']['contact'] if node in self.addresses['nodes'] and 'owner' in self.addresses['nodes'][node]['nodeinfo'] else '' + mail = 'JA' if (node in self.addresses['nodes'] and 'owner' in self.addresses['nodes'][node]['nodeinfo']) or (node in self.addressesOld['nodes'] and 'owner' in self.addressesOld['nodes'][node]['nodeinfo']) else 'NEIN' + print date +'\t'+ val['lastseen'] + '\t' + dest_dom + '\t' + val['pub_v6'] + '\t' + node + '\t' + val['name'] + '\t' + mail + + + def printResults(self): + ordered = collections.OrderedDict(sorted(self.results.items())) + print "date\tnodes_online\tusers_online" + for k,v in ordered.iteritems(): + print k+'\t'+str(v['nodes_online'])+'\t'+str(v['users_online']) + + + def getPublicAddress(self,node): + if 'addresses' in node['nodeinfo']['network']: + for address in node['nodeinfo']['network']['addresses']: + if address.startswith('2a03'): + return address + return None + +dmax = OfflineChecker('http://karte.freifunk-muensterland.org/data/nodes.json') \ No newline at end of file diff --git a/hieraException.py b/hieraException.py new file mode 100644 index 0000000..bab98ce --- /dev/null +++ b/hieraException.py @@ -0,0 +1,7 @@ +#!/usr/bin/python +# -*- coding: utf-8 - +#Imports: + + +class HieraException(Exception): + pass \ No newline at end of file diff --git a/node_hierarchy.py b/node_hierarchy.py index 1c4decb..ae4db5a 100755 --- a/node_hierarchy.py +++ b/node_hierarchy.py @@ -1,97 +1,52 @@ #!/usr/bin/python # -*- coding: utf-8 - #Imports: +import argparse, json, sys from domain_selector import DomainSelector +from hieraException import HieraException + +parser = argparse.ArgumentParser(description='This Script generates a hierarchical nodes list for node migration using nginx geo feature.') +parser.add_argument('--json-path', required=False, default='https://service.freifunk-muensterland.de/maps/data/', help='Path of nodes.json and graph.json (can be local folder or remote URL).') +parser.add_argument('--targets-file', required=False, help='Json file of targets for nominatim geocoder.', default='./targets.json') +parser.add_argument('-t', '--targets', nargs='*', required=False, help='List of target names from target-file which should be proceeded. Example: -t citya -t cityb ...') +parser.add_argument('-a', '--all', '--all-targets', required=False, help='Proceed all targets from targets file.', action='store_true') +parser.add_argument('--out-path', required=False, help='Directory where the generated Output should stored.', default='./webserver-configuration/') +parser.add_argument('--only-specific-branch', required=False, help='Only attend nodes from specific branch.', default=None) +parser.add_argument('-p', '--print-status', required=False, action='store_true', help='Print Status (like geocoder tasks).') +args = parser.parse_args() + +def prepareTargets(args): + + resource = open(args.targets_file) + targets = json.loads(resource.read()) + resource.close() + + if len(targets) == 0: + print "\033[91mError:\033[0m No targets were found in targets file." + sys.exit(1) + if args.all == True: + return targets + elif args.targets == None or len(args.targets) == 0: + print "\033[91mError:\033[0m No target was given as argument and even --all switch was not enabled." + sys.exit(1) + else: + specific_targets = {} + for k, v in targets.iteritems(): + if k in args.targets: + specific_targets[k] = v + return specific_targets + + + +print args + +targets = prepareTargets(args) + -targets = { -# 'muenster' : [ -# {'city' : u'Münster'}, -# {'county' : u'Münster'}, -# ], - # 'kreis_warendorf' : [ - # {'county' : u'Kreis Warendorf'}, - # ], - # 'kreis_coesfeld' : [ - # {'county' : u'Kreis Coesfeld'}, - # ], - # 'kreis_steinfurt_west' : [ - # {'town' : u'48565'}, - # {'village' : u'Wettringen'}, - # {'town' : u'Ochtrup'}, - # {'village' : u'Metelen'}, - # {'town' : u'Horstmar'}, - # {'village' : u'Laer'}, - # {'village' : u'Nordwalde'}, - # {'village' : u'Altenberge'}, - # ], - # 'kreis_steinfurt_ost' : [ - # {'town' : u'Emsdetten'}, - # {'town' : u'Neuenkirchen'}, - # {'town' : u'Rheine'}, - # {'town' : u'Greven'}, - # {'village' : u'Ladbergen'}, - # {'town' : u'Lengerich'}, - # {'town' : u'Tecklenburg'}, - # {'village' : u'Lienen'}, - # ], - # 'muenster_stadt' : [ - # {'city_district' : u'Münster-Mitte'}, - # {'city_district' : u'Münster-Nord'}, - # {'city_district' : u'Münster-Ost'}, - # {'suburb' : u'Berg Fidel'}, - # {'suburb' : u'Gremmendorf'}, - # {'suburb' : u'Mecklenbeck'}, - # {'suburb' : u'Gievenbeck'}, - # {'suburb' : u'Nienberge'}, - # {'suburb' : u'Roxel'}, - # {'suburb' : u'Sentruper Höhe'}, - # ], - # 'muenster_sued' : [ - # {'suburb' : u'Amelsbüren'}, - # {'suburb' : u'Hiltrup'}, - # {'suburb' : u'Albachten'}, - # ], - # 'kreis_borken' : [ - # {'town' : u'Ahaus'}, - # {'town' : u'Bocholt'}, - # {'town' : u'Borken'}, - # {'town' : u'Gescher'}, - # {'village' : u'Heek'}, - # {'town' : u'Heiden'}, - # {'town' : u'Isselburg'}, - # {'village' : u'Legden'}, - # {'town' : u'Raesfeld'}, - # {'town' : u'Reken'}, - # {'town' : u'Rhede'}, - # {'village' : u'Schöppingen'}, - # {'town' : u'Stadtlohn'}, - # {'village' : u'Südlohn'}, - # {'town' : u'Velen'}, - # {'town' : u'Vreden'}, - # ], - # 'sassenberg' : [ - # {'town' : u'Sassenberg'}, - # ], - # 'telgte' : [ - # {'town' : u'Telgte'}, - # ], - # 'warendorf_stadt' : [ - # {'town' : u'Warendorf'}, - # ] - 'stadt_stadtlohn' : [ - {'town' : u'Stadtlohn'}, - ], - 'stadt_bocholt' : [ - {'town' : u'Bocholt'}, - ], - 'stadt_telgte' : [ - {'town' : u'Telgte'}, - ], - 'stadt_warendorf' : [ - {'town' : u'Warendorf'}, - ], -} #ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) #ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None) -ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None) +try: + ds = DomainSelector(nodesFile = args.json_path.rstrip('/')+'/nodes.json', graphFile = args.json_path.rstrip('/')+'/graph.json', printStatus = args.print_status, dataPath = args.out_path, targets = targets, branch = args.only_specific_branch) +except HieraException: + print "\033[93mFailed:\033[0m Process was interrupted by HieraException-Exception (see error messages above)." \ No newline at end of file diff --git a/targets.json b/targets.json new file mode 100644 index 0000000..bde6619 --- /dev/null +++ b/targets.json @@ -0,0 +1,75 @@ +{ + "domaene_01" : [ + {"city_district" : "Münster-Mitte"}, + {"city_district" : "Münster-Nord"}, + {"city_district" : "Münster-Ost"}, + {"suburb" : "Berg Fidel"}, + {"suburb" : "Gremmendorf"}, + {"suburb" : "Mecklenbeck"}, + {"suburb" : "Gievenbeck"}, + {"suburb" : "Nienberge"}, + {"suburb" : "Roxel"}, + {"suburb" : "Sentruper Höhe"} + ], + "domaene_02" : [ + {"county" : "Kreis Coesfeld"} + ], + "domaene_03" : [ + {"town" : "48565"}, + {"village" : "Wettringen"}, + {"town" : "Ochtrup"}, + {"village" : "Metelen"}, + {"town" : "Horstmar"}, + {"village" : "Laer"}, + {"village" : "Nordwalde"}, + {"village" : "Altenberge"} + ], + "domaene_04" : [ + {"town" : "Emsdetten"}, + {"town" : "Neuenkirchen"}, + {"town" : "Rheine"}, + {"town" : "Greven"}, + {"village" : "Ladbergen"}, + {"town" : "Lengerich"}, + {"town" : "Tecklenburg"}, + {"village" : "Lienen"} + ], + "domaene_05" : [ + {"suburb" : "Amelsbüren"}, + {"suburb" : "Hiltrup"}, + {"suburb" : "Albachten"} + ], + "domaene_06" : [ + {"town" : "Ahaus"}, + {"town" : "Bocholt"}, + {"town" : "Borken"}, + {"town" : "Gescher"}, + {"village" : "Heek"}, + {"town" : "Heiden"}, + {"town" : "Isselburg"}, + {"village" : "Legden"}, + {"town" : "Raesfeld"}, + {"town" : "Reken"}, + {"town" : "Rhede"}, + {"village" : "Schöppingen"}, + {"town" : "Stadtlohn"}, + {"village" : "Südlohn"}, + {"town" : "Velen"}, + {"town" : "Vreden"} + ], + "domaene_07" : [ + {"town" : "Telgte"} + ], + "domaene_09" : [ + {"town" : "Stadtlohn"} + ], + "domaene_11" : [ + {"town" : "Bocholt"} + ], + "stadt_warendorf" : [ + {"town" : "Warendorf"} + ], + "domaene_14" : [ + {"county" : "Kreis Warendorf"} + ] +} \ No newline at end of file From fac219a7acca43025b4f206d3c15627f124c0a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Tue, 29 Mar 2016 17:59:35 +0200 Subject: [PATCH 09/11] Updated mechanism for generating output for nodes with disabled autoupdater or no geoinformations --- domain_selector.py | 37 +++++++++++++++++++++++-------------- graph.py | 38 +++++++++++++++++++++++++++++--------- node_hierarchy.py | 8 -------- 3 files changed, 52 insertions(+), 31 deletions(-) diff --git a/domain_selector.py b/domain_selector.py index 76c0f55..6464466 100644 --- a/domain_selector.py +++ b/domain_selector.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 - #Imports: -import json, urllib, os +import json, urllib, os, glob from graph import Graph from hieraException import HieraException @@ -13,9 +13,11 @@ class DomainSelector: self.printStatus = printStatus self.targets = targets + self.dataPath = dataPath.rstrip('/') self.nodesData = self.__getFile__(nodesFile) self.graphData = self.__getFile__(graphFile) - self.dataPath = dataPath + + self.__prepareOutDir__() self.graph = Graph(self.nodesData, self.graphData) if self.targets == None: @@ -28,8 +30,14 @@ class DomainSelector: self.writeConfigFiles(nodes,k) self.writeDumpFile(nodes,k) nodes = {} - self.writeConfigFiles(self.graph.nodes_no_autoupdater,"no_autoupdater") - self.writeConfigFiles(self.graph.nodes_no_geo,"no_geo") + self.writeConfigFiles(self.graph.getProblemNodes(noAutoupdater = True),"no_autoupdater") + self.writeConfigFiles(self.graph.getProblemNodes(noGeodata = True),"no_geo") + self.writeConfigFiles(self.graph.getProblemNodes(noGeodata = True, noAutoupdater = True),"no_nothing") + + def __prepareOutDir__(self): + files = glob.glob(self.dataPath+'/*') + for f in files: + os.remove(f) def __getFile__(self, nodesFile): if nodesFile.startswith('https://') or nodesFile.startswith('http://'): @@ -52,16 +60,17 @@ class DomainSelector: def writeConfigFiles(self, nodes, name): maxDepth = self.maxDepth(nodes) - for i in range(0,maxDepth): - content = 'geo $switch {\n default 0;' - f = open(self.dataPath.rstrip('/')+'/'+name+'_node_level'+str(i),'w') - for node in nodes.itervalues(): - if node.stepsToVpn == i: - if node.ipv6 and node.hostname: - content += '\n '+node.ipv6+' 1; #'+node.hostname - content += '\n}' - f.write(content.encode('utf8')) - f.close() + if len(nodes) > 0: + for i in range(0,maxDepth): + content = 'geo $switch {\n default 0;' + f = open(self.dataPath.rstrip('/')+'/'+name+'_node_level'+str(i),'w') + for node in nodes.itervalues(): + if node.stepsToVpn == i: + if node.ipv6 and node.hostname: + content += '\n '+node.ipv6+' 1; #'+node.hostname + content += '\n}' + f.write(content.encode('utf8')) + f.close() def writeDumpFile(self, nodes, name): content = {} diff --git a/graph.py b/graph.py index 39dad55..c5ba268 100644 --- a/graph.py +++ b/graph.py @@ -113,25 +113,45 @@ class Graph: def getOnlineState(self,node_id): return self.nodes['nodes'][node_id]['flags']['online'] + def getProblemNodes(self, noAutoupdater = False, noGeodata = False, online = True): + results = {} + for k,v in self.nodes_list.iteritems(): + if v.isOnline or online == False: + if noAutoupdater and noGeodata: + if not v.autoupdater and not v.geodata: + results[k] = v + elif noAutoupdater: + if v.autoupdater and v.geodata: + results[k] = v + elif noGeodata: + if not v.geodata and v.autoupdater: + results[k] = v + return results + def getNodeCloudsIn(self, region, branch = 'stable'): results = {} - noAuto = False +# noAuto = False for k,v in self.getAllLevelXNodes(0).iteritems(): if v.isOnline == True: if v.geodata != None: if v.isInRegion(region): - noAuto = False for ksub,vsub in v.getNodeCloud({}).iteritems(): if not vsub.autoupdater or (branch and vsub.branch != branch): - #break - noAuto = True - self.nodes_no_autoupdater[ksub] = vsub - #else: - if not noAuto: + break + else: results.update(v.getNodeCloud({})) - else: - self.nodes_no_geo.update(v.getNodeCloud({})) +# noAuto = False +# for ksub,vsub in v.getNodeCloud({}).iteritems(): +# if not vsub.autoupdater or (branch and vsub.branch != branch): +# #break +# noAuto = True +# self.nodes_no_autoupdater[ksub] = vsub +# #else: +# if not noAuto: +# results.update(v.getNodeCloud({})) +# else: +# self.nodes_no_geo.update(v.getNodeCloud({})) print "Result:",len(results), region return results diff --git a/node_hierarchy.py b/node_hierarchy.py index ae4db5a..6411ba4 100755 --- a/node_hierarchy.py +++ b/node_hierarchy.py @@ -36,16 +36,8 @@ def prepareTargets(args): specific_targets[k] = v return specific_targets - - -print args - targets = prepareTargets(args) - - -#ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) -#ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None) try: ds = DomainSelector(nodesFile = args.json_path.rstrip('/')+'/nodes.json', graphFile = args.json_path.rstrip('/')+'/graph.json', printStatus = args.print_status, dataPath = args.out_path, targets = targets, branch = args.only_specific_branch) except HieraException: From 222bd388a3cb86d5b589f46c0e661a0fa502c65f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Sun, 3 Apr 2016 13:14:51 +0200 Subject: [PATCH 10/11] Removed comments. Fixed minor issue. --- graph.py | 11 ----------- node.py | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/graph.py b/graph.py index c5ba268..7492ed0 100644 --- a/graph.py +++ b/graph.py @@ -141,17 +141,6 @@ class Graph: break else: results.update(v.getNodeCloud({})) -# noAuto = False -# for ksub,vsub in v.getNodeCloud({}).iteritems(): -# if not vsub.autoupdater or (branch and vsub.branch != branch): -# #break -# noAuto = True -# self.nodes_no_autoupdater[ksub] = vsub -# #else: -# if not noAuto: -# results.update(v.getNodeCloud({})) -# else: -# self.nodes_no_geo.update(v.getNodeCloud({})) print "Result:",len(results), region return results diff --git a/node.py b/node.py index 632de18..1fbe8ef 100644 --- a/node.py +++ b/node.py @@ -105,4 +105,4 @@ class Node(object): if result['cached'] == False: time.sleep(1) else: - self['geodata'] = None \ No newline at end of file + self.getodata = None \ No newline at end of file From 1eb13ac4e6368458c866d02ce696e4d89704ef6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20W=C3=BCllhorst?= Date: Sun, 3 Apr 2016 13:27:17 +0200 Subject: [PATCH 11/11] Added domaene_08, domaene_12 and domaene_13 to targets.json --- targets.json | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/targets.json b/targets.json index bde6619..c97c0a6 100644 --- a/targets.json +++ b/targets.json @@ -60,16 +60,32 @@ "domaene_07" : [ {"town" : "Telgte"} ], + "domaene_08" : [ + {"town" : "Gescher"} + ], "domaene_09" : [ {"town" : "Stadtlohn"} ], "domaene_11" : [ {"town" : "Bocholt"} ], + "domaene_12" : [ + { + "town" : "Dülmen", + "suburb" : "Mitte" + }, + {"suburb" : "Hausdülmen"}, + {"suburb" : "Merfeld"}, + {"suburb" : "Buldern"}, + {"suburb" : "Hiddingsel"} + ], + "domaene_13" : [ + {"suburb" : "Rorup"} + ], "stadt_warendorf" : [ {"town" : "Warendorf"} ], "domaene_14" : [ {"county" : "Kreis Warendorf"} ] -} \ No newline at end of file +}