diff --git a/domain_selector.py b/domain_selector.py index 1891e39..5b83fa6 100644 --- a/domain_selector.py +++ b/domain_selector.py @@ -22,6 +22,8 @@ class DomainSelector: self.writeConfigFiles(nodes,k) self.writeDumpFile(nodes,k) nodes = {} + self.writeConfigFiles(self.graph.nodes_no_autoupdater,"no_autoupdater") + self.writeConfigFiles(self.graph.nodes_no_geo,"no_geo") def __getFile__(self, nodesFile): if nodesFile.startswith('https://') or nodesFile.startswith('http://'): @@ -39,14 +41,12 @@ class DomainSelector: def writeConfigFiles(self, nodes, name): maxDepth = self.maxDepth(nodes) for i in range(0,maxDepth): - content = 'geo $switch {\n\tdefault\t0;' + content = 'geo $switch {\n default 0;' f = open(self.dataPath+'/'+name+'_node_level'+str(i),'w') for node in nodes.itervalues(): if node.stepsToVpn == i: if node.ipv6 and node.hostname: - content += '\n\t'+node.ipv6+'\t1;\t #'+node.hostname - #else: - # print node.nodeid + content += '\n '+node.ipv6+' 1; #'+node.hostname content += '\n}' f.write(content.encode('utf8')) f.close() @@ -69,4 +69,4 @@ class DomainSelector: for v in nodes.itervalues(): if v.stepsToVpn > maxDepth: maxDepth = v.stepsToVpn - return maxDepth+1 \ No newline at end of file + return maxDepth+1 diff --git a/graph.py b/graph.py index 99fcffc..bb090e9 100644 --- a/graph.py +++ b/graph.py @@ -13,6 +13,8 @@ class Graph: self.data = graphData self.nodes = nodesData self.nodes_list = {} + self.nodes_no_autoupdater = {} + self.nodes_no_geo = {} self.parseNodes() self.parseLinks() self.calculateStepsToVpn() @@ -21,14 +23,17 @@ class Graph: def parseNodes(self): for k,v in self.nodes['nodes'].iteritems(): lat, lon = self.getGeo(k) - node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder, autoupdater = self.getAutoupdaterStatus(k), branch = self.getBranch(k)) + node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder, autoupdater = self.getAutoupdaterStatus(k), branch = self.getBranch(k), isGateway = self.getIsGateway(k)) self.nodes_list[k] = node def parseLinks(self): link_nodes = self.data['batadv']['nodes'] for link in self.data['batadv']['links']: if 'node_id' in link_nodes[link['source']].keys() and 'node_id' in link_nodes[link['target']].keys():#else it is a vpn link - self.setLinkBetween(link_nodes[link['source']]['node_id'], link_nodes[link['target']]['node_id']) + if self.nodes_list[link_nodes[link['source']]['node_id']].isGateway == True or self.nodes_list[link_nodes[link['target']]['node_id']].isGateway: + self.setVpnLink(link['source'], link['target']) + else: + self.setLinkBetween(link_nodes[link['source']]['node_id'], link_nodes[link['target']]['node_id']) else: self.setVpnLink(link['source'], link['target']) @@ -46,11 +51,11 @@ class Graph: } def setVpnLink(self, src, dst): - if 'node_id' not in self.data['batadv']['nodes'][src].keys(): - if self.data['batadv']['nodes'][dst]['node_id']: + if 'node_id' not in self.data['batadv']['nodes'][src].keys() or self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].isGateway == True: + if 'node_id' in self.data['batadv']['nodes'][dst]: self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].stepsToVpn = 0 - elif 'node_id' not in self.data['batadv']['nodes'][dst].keys(): - if self.data['batadv']['nodes'][src]['node_id']: + elif 'node_id' not in self.data['batadv']['nodes'][dst].keys() or self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].isGateway == True: + if 'node_id' in self.data['batadv']['nodes'][src]: self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].stepsToVpn = 0 def calculateStepsToVpn(self): @@ -73,6 +78,9 @@ class Graph: def getHostname(self,node_id): return self.nodes['nodes'][node_id]['nodeinfo']['hostname'] + def getIsGateway(self,node_id): + return self.nodes['nodes'][node_id]['flags']['gateway'] + def getAutoupdaterStatus(self, node_id): #return True if 'autoupdater' in self.nodes['nodes'][node_id]['nodeinfo']['software']: @@ -108,14 +116,22 @@ class Graph: def getNodeCloudsIn(self, region, branch = 'stable'): results = {} + noAuto = False for k,v in self.getAllLevelXNodes(0).iteritems(): - if v.geodata != None and v.isOnline == True: - if v.isInRegion(region): - for ksub,vsub in v.getNodeCloud({}).iteritems(): - if not vsub.autoupdater or vsub.branch != branch: - break - else: - results.update(v.getNodeCloud({})) + if v.isOnline == True: + if v.geodata != None: + if v.isInRegion(region): + noAuto = False + for ksub,vsub in v.getNodeCloud({}).iteritems(): + if not vsub.autoupdater or (branch and vsub.branch != branch): + #break + noAuto = True + self.nodes_no_autoupdater[ksub] = vsub + #else: + if not noAuto: + results.update(v.getNodeCloud({})) + else: + self.nodes_no_geo.update(v.getNodeCloud({})) print "Result:",len(results), region return results diff --git a/no_coords.py b/no_coords.py new file mode 100755 index 0000000..3b9093e --- /dev/null +++ b/no_coords.py @@ -0,0 +1,116 @@ +#!/usr/bin/python +# +# (c) 2016 descilla +# +# This script is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License or any later version. +# +# This script is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY. See the +# GNU General Public License for more details. +# +# For a copy of the GNU General Public License +# see . +# +import glob, os, json, collections, argparse, urllib, datetime +from collections import OrderedDict + +class OfflineChecker: + def __init__(self, dataFile): + self.printStatus = False + self.dataSet = self.__getFile__(dataFile) + self.fileNames = [] + self.results = {} + self.addresses = [] + self.getAddressFiles() + self.getFwState() + + def __getFile__(self, nodesFile): + if nodesFile.startswith('https://') or nodesFile.startswith('http://'): + if self.printStatus: + print "Download node.json from URL: " + nodesFile + resource = urllib.urlopen(nodesFile) + else: + if self.printStatus: + print "Open node.json file: " + nodesFile + resource = open(nodesFile) + data = json.loads(resource.read()) + resource.close() + return data + + def getAddressFiles(self): + for file in glob.iglob('./nodes_adresses_*'): + self.addresses.append(self.__getFile__(file)) + + def readFile(self): + fil = 'operate.txt' + results = [] + with open(fil) as lg: + for line in lg: + results.append(line) + return results + + def getNodeAddressItem(self,ipv6): + pub = "" + i = 0 + for adr in self.addresses: + i +=1 + for val in adr['nodes'].itervalues(): + pub = self.getPublicAddress(val) + if pub and pub in ipv6: + if 'owner' in val['nodeinfo']: + return val + + def getNodeItem(self,ipv6): + pub = "" + for val in self.dataSet['nodes'].itervalues(): + pub = self.getPublicAddress(val) + if pub and pub in ipv6: + return val + + def getFwState(self): + lastDay = datetime.datetime.today() - datetime.timedelta(hours = 48) + onlyoldernodes = False + results = {} + for nodeIP in self.readFile(): + nodeAddresses = self.getNodeAddressItem(nodeIP) + node = self.getNodeItem(nodeIP) + if node: + nodeLastSeen = datetime.datetime.strptime(node['lastseen'],'%Y-%m-%dT%H:%M:%S') + if nodeLastSeen < lastDay or onlyoldernodes == False: + au = node['nodeinfo']['software']['autoupdater']['branch'] + loca = 'JA' if 'location' in node['nodeinfo'] else 'NEIN' + if nodeAddresses: + mail = nodeAddresses['nodeinfo']['owner']['contact'] if 'owner' in nodeAddresses['nodeinfo'] else 'NEIN' + #mail = 'JA' if 'owner' in nodeAddresses['nodeinfo'] else 'NEIN' + else: + mail = 'NEIN' + results[node['nodeinfo']['node_id']] = { + 'lastseen' : node['lastseen'], + 'ipv6' : self.getPublicAddress(node), + 'node_id' : node['nodeinfo']['node_id'], + 'name' : node['nodeinfo']['hostname'], + 'contact' : mail, + 'fw_base' : node['nodeinfo']['software']['firmware']['base'], + 'fw_release' : node['nodeinfo']['software']['firmware']['release'], + 'au_enabled' : str(node['nodeinfo']['software']['autoupdater']['enabled']), + 'au_branch' : au, + 'router_modell' : node['nodeinfo']['hardware']['model'], + 'geo' : loca, + } + #print node['lastseen'] + ';' + self.getPublicAddress(node) + ';' + node['nodeinfo']['node_id'] + ';' + node['nodeinfo']['hostname'] + ';' + mail + ';' + node['nodeinfo']['software']['firmware']['base'] + ';' + node['nodeinfo']['software']['firmware']['release'] + ';' + str(node['nodeinfo']['software']['autoupdater']['enabled']) + ';' + au + ';' + node['nodeinfo']['hardware']['model'] + ';' + loca + self.printCSV(results) + def printCSV(self, data): + od = OrderedDict(sorted(data.items(), key=lambda x: x[1]['lastseen'], reverse=True)) + print 'zuletzt online;nodeid;Knotenname;mailaddress;Firmware Base;Firmware Release;Autoupdater;AU-Branch;Router-Modell;geo' + for item in od.itervalues(): + print item['lastseen'] + ';' + item['node_id'] + ';' + item['name'] + ';' + item['contact'] + ';' + item['fw_base'] + ';' + item['fw_release'] + ';' + item['au_enabled'] + ';' + item['au_branch'] + ';' + item['router_modell'] + ';' + item['geo'] + def getPublicAddress(self,node): + if 'addresses' in node['nodeinfo']['network']: + for address in node['nodeinfo']['network']['addresses']: + if address.startswith('2a03'): + return address + return None + +dmax = OfflineChecker('https://service.freifunk-muensterland.de/maps/data/nodes.json') \ No newline at end of file diff --git a/node.py b/node.py index a67718b..632de18 100644 --- a/node.py +++ b/node.py @@ -4,7 +4,7 @@ from geocode import Geocode import time class Node(object): - def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None, autoupdater = False, branch = None): + def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None, autoupdater = False, branch = None, isGateway = False): self.coder = coder if self.coder == None: self.coder = Geocode(geocoderCache = True, printStatus = True) @@ -19,6 +19,7 @@ class Node(object): self.branch = branch self._geo = None self.geodata = None + self.isGateway = isGateway if lat != None and lon != None: self.geo = { 'lat' : lat, diff --git a/node_hierarchy.py b/node_hierarchy.py index abe1170..1c4decb 100755 --- a/node_hierarchy.py +++ b/node_hierarchy.py @@ -8,77 +8,90 @@ targets = { # {'city' : u'Münster'}, # {'county' : u'Münster'}, # ], - 'kreis_warendorf' : [ - {'county' : u'Kreis Warendorf'}, - ], - 'kreis_coesfeld' : [ - {'county' : u'Kreis Coesfeld'}, - ], - 'kreis_steinfurt_west' : [ - {'town' : u'48565'}, - {'village' : u'Wettringen'}, - {'town' : u'Ochtrup'}, - {'village' : u'Metelen'}, - {'town' : u'Horstmar'}, - {'village' : u'Laer'}, - {'village' : u'Nordwalde'}, - {'village' : u'Altenberge'}, - ], - 'kreis_steinfurt_ost' : [ - {'town' : u'Emsdetten'}, - {'town' : u'Neuenkirchen'}, - {'town' : u'Rheine'}, - {'town' : u'Greven'}, - {'village' : u'Ladbergen'}, - {'town' : u'Lengerich'}, - {'town' : u'Tecklenburg'}, - {'village' : u'Lienen'}, - ], - 'muenster_stadt' : [ - {'city_district' : u'Münster-Mitte'}, - {'city_district' : u'Münster-Nord'}, - {'city_district' : u'Münster-Ost'}, - {'suburb' : u'Berg Fidel'}, - {'suburb' : u'Gremmendorf'}, - {'suburb' : u'Mecklenbeck'}, - {'suburb' : u'Gievenbeck'}, - {'suburb' : u'Nienberge'}, - {'suburb' : u'Roxel'}, - {'suburb' : u'Sentruper Höhe'}, - ], - 'muenster_sued' : [ - {'suburb' : u'Amelsbüren'}, - {'suburb' : u'Hiltrup'}, - {'suburb' : u'Albachten'}, - ], - 'kreis_borken' : [ - {'town' : u'Ahaus'}, - {'town' : u'Bocholt'}, - {'town' : u'Borken'}, - {'town' : u'Gescher'}, - {'village' : u'Heek'}, - {'town' : u'Heiden'}, - {'town' : u'Isselburg'}, - {'village' : u'Legden'}, - {'town' : u'Raesfeld'}, - {'town' : u'Reken'}, - {'town' : u'Rhede'}, - {'village' : u'Schöppingen'}, + # 'kreis_warendorf' : [ + # {'county' : u'Kreis Warendorf'}, + # ], + # 'kreis_coesfeld' : [ + # {'county' : u'Kreis Coesfeld'}, + # ], + # 'kreis_steinfurt_west' : [ + # {'town' : u'48565'}, + # {'village' : u'Wettringen'}, + # {'town' : u'Ochtrup'}, + # {'village' : u'Metelen'}, + # {'town' : u'Horstmar'}, + # {'village' : u'Laer'}, + # {'village' : u'Nordwalde'}, + # {'village' : u'Altenberge'}, + # ], + # 'kreis_steinfurt_ost' : [ + # {'town' : u'Emsdetten'}, + # {'town' : u'Neuenkirchen'}, + # {'town' : u'Rheine'}, + # {'town' : u'Greven'}, + # {'village' : u'Ladbergen'}, + # {'town' : u'Lengerich'}, + # {'town' : u'Tecklenburg'}, + # {'village' : u'Lienen'}, + # ], + # 'muenster_stadt' : [ + # {'city_district' : u'Münster-Mitte'}, + # {'city_district' : u'Münster-Nord'}, + # {'city_district' : u'Münster-Ost'}, + # {'suburb' : u'Berg Fidel'}, + # {'suburb' : u'Gremmendorf'}, + # {'suburb' : u'Mecklenbeck'}, + # {'suburb' : u'Gievenbeck'}, + # {'suburb' : u'Nienberge'}, + # {'suburb' : u'Roxel'}, + # {'suburb' : u'Sentruper Höhe'}, + # ], + # 'muenster_sued' : [ + # {'suburb' : u'Amelsbüren'}, + # {'suburb' : u'Hiltrup'}, + # {'suburb' : u'Albachten'}, + # ], + # 'kreis_borken' : [ + # {'town' : u'Ahaus'}, + # {'town' : u'Bocholt'}, + # {'town' : u'Borken'}, + # {'town' : u'Gescher'}, + # {'village' : u'Heek'}, + # {'town' : u'Heiden'}, + # {'town' : u'Isselburg'}, + # {'village' : u'Legden'}, + # {'town' : u'Raesfeld'}, + # {'town' : u'Reken'}, + # {'town' : u'Rhede'}, + # {'village' : u'Schöppingen'}, + # {'town' : u'Stadtlohn'}, + # {'village' : u'Südlohn'}, + # {'town' : u'Velen'}, + # {'town' : u'Vreden'}, + # ], + # 'sassenberg' : [ + # {'town' : u'Sassenberg'}, + # ], + # 'telgte' : [ + # {'town' : u'Telgte'}, + # ], + # 'warendorf_stadt' : [ + # {'town' : u'Warendorf'}, + # ] + 'stadt_stadtlohn' : [ {'town' : u'Stadtlohn'}, - {'village' : u'Südlohn'}, - {'town' : u'Velen'}, - {'town' : u'Vreden'}, ], - 'sassenberg' : [ - {'town' : u'Sassenberg'}, + 'stadt_bocholt' : [ + {'town' : u'Bocholt'}, ], - 'telgte' : [ + 'stadt_telgte' : [ {'town' : u'Telgte'}, ], - 'warendorf_stadt' : [ + 'stadt_warendorf' : [ {'town' : u'Warendorf'}, - ] + ], } #ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets) -ds = DomainSelector(nodesFile = 'https://freifunk-muensterland.de/map/data/nodes.json', graphFile = 'https://freifunk-muensterland.de/map/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = 'stable') \ No newline at end of file +#ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None) +ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None)