- Added support of GW-mesh-links (Domaene 14)
- Added output of nodes without geo or disabled autoupdater - Changed output styling - Several minor changes
This commit is contained in:
Simon Wüllhorst
parent
330e78c85f
commit
bd1798195b
|
|
@ -22,6 +22,8 @@ class DomainSelector:
|
|||
self.writeConfigFiles(nodes,k)
|
||||
self.writeDumpFile(nodes,k)
|
||||
nodes = {}
|
||||
self.writeConfigFiles(self.graph.nodes_no_autoupdater,"no_autoupdater")
|
||||
self.writeConfigFiles(self.graph.nodes_no_geo,"no_geo")
|
||||
|
||||
def __getFile__(self, nodesFile):
|
||||
if nodesFile.startswith('https://') or nodesFile.startswith('http://'):
|
||||
|
|
@ -39,14 +41,12 @@ class DomainSelector:
|
|||
def writeConfigFiles(self, nodes, name):
|
||||
maxDepth = self.maxDepth(nodes)
|
||||
for i in range(0,maxDepth):
|
||||
content = 'geo $switch {\n\tdefault\t0;'
|
||||
content = 'geo $switch {\n default 0;'
|
||||
f = open(self.dataPath+'/'+name+'_node_level'+str(i),'w')
|
||||
for node in nodes.itervalues():
|
||||
if node.stepsToVpn == i:
|
||||
if node.ipv6 and node.hostname:
|
||||
content += '\n\t'+node.ipv6+'\t1;\t #'+node.hostname
|
||||
#else:
|
||||
# print node.nodeid
|
||||
content += '\n '+node.ipv6+' 1; #'+node.hostname
|
||||
content += '\n}'
|
||||
f.write(content.encode('utf8'))
|
||||
f.close()
|
||||
|
|
@ -69,4 +69,4 @@ class DomainSelector:
|
|||
for v in nodes.itervalues():
|
||||
if v.stepsToVpn > maxDepth:
|
||||
maxDepth = v.stepsToVpn
|
||||
return maxDepth+1
|
||||
return maxDepth+1
|
||||
|
|
|
|||
42
graph.py
42
graph.py
|
|
@ -13,6 +13,8 @@ class Graph:
|
|||
self.data = graphData
|
||||
self.nodes = nodesData
|
||||
self.nodes_list = {}
|
||||
self.nodes_no_autoupdater = {}
|
||||
self.nodes_no_geo = {}
|
||||
self.parseNodes()
|
||||
self.parseLinks()
|
||||
self.calculateStepsToVpn()
|
||||
|
|
@ -21,14 +23,17 @@ class Graph:
|
|||
def parseNodes(self):
|
||||
for k,v in self.nodes['nodes'].iteritems():
|
||||
lat, lon = self.getGeo(k)
|
||||
node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder, autoupdater = self.getAutoupdaterStatus(k), branch = self.getBranch(k))
|
||||
node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder, autoupdater = self.getAutoupdaterStatus(k), branch = self.getBranch(k), isGateway = self.getIsGateway(k))
|
||||
self.nodes_list[k] = node
|
||||
|
||||
def parseLinks(self):
|
||||
link_nodes = self.data['batadv']['nodes']
|
||||
for link in self.data['batadv']['links']:
|
||||
if 'node_id' in link_nodes[link['source']].keys() and 'node_id' in link_nodes[link['target']].keys():#else it is a vpn link
|
||||
self.setLinkBetween(link_nodes[link['source']]['node_id'], link_nodes[link['target']]['node_id'])
|
||||
if self.nodes_list[link_nodes[link['source']]['node_id']].isGateway == True or self.nodes_list[link_nodes[link['target']]['node_id']].isGateway:
|
||||
self.setVpnLink(link['source'], link['target'])
|
||||
else:
|
||||
self.setLinkBetween(link_nodes[link['source']]['node_id'], link_nodes[link['target']]['node_id'])
|
||||
else:
|
||||
self.setVpnLink(link['source'], link['target'])
|
||||
|
||||
|
|
@ -46,11 +51,11 @@ class Graph:
|
|||
}
|
||||
|
||||
def setVpnLink(self, src, dst):
|
||||
if 'node_id' not in self.data['batadv']['nodes'][src].keys():
|
||||
if self.data['batadv']['nodes'][dst]['node_id']:
|
||||
if 'node_id' not in self.data['batadv']['nodes'][src].keys() or self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].isGateway == True:
|
||||
if 'node_id' in self.data['batadv']['nodes'][dst]:
|
||||
self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].stepsToVpn = 0
|
||||
elif 'node_id' not in self.data['batadv']['nodes'][dst].keys():
|
||||
if self.data['batadv']['nodes'][src]['node_id']:
|
||||
elif 'node_id' not in self.data['batadv']['nodes'][dst].keys() or self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].isGateway == True:
|
||||
if 'node_id' in self.data['batadv']['nodes'][src]:
|
||||
self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].stepsToVpn = 0
|
||||
|
||||
def calculateStepsToVpn(self):
|
||||
|
|
@ -73,6 +78,9 @@ class Graph:
|
|||
def getHostname(self,node_id):
|
||||
return self.nodes['nodes'][node_id]['nodeinfo']['hostname']
|
||||
|
||||
def getIsGateway(self,node_id):
|
||||
return self.nodes['nodes'][node_id]['flags']['gateway']
|
||||
|
||||
def getAutoupdaterStatus(self, node_id):
|
||||
#return True
|
||||
if 'autoupdater' in self.nodes['nodes'][node_id]['nodeinfo']['software']:
|
||||
|
|
@ -108,14 +116,22 @@ class Graph:
|
|||
|
||||
def getNodeCloudsIn(self, region, branch = 'stable'):
|
||||
results = {}
|
||||
noAuto = False
|
||||
for k,v in self.getAllLevelXNodes(0).iteritems():
|
||||
if v.geodata != None and v.isOnline == True:
|
||||
if v.isInRegion(region):
|
||||
for ksub,vsub in v.getNodeCloud({}).iteritems():
|
||||
if not vsub.autoupdater or vsub.branch != branch:
|
||||
break
|
||||
else:
|
||||
results.update(v.getNodeCloud({}))
|
||||
if v.isOnline == True:
|
||||
if v.geodata != None:
|
||||
if v.isInRegion(region):
|
||||
noAuto = False
|
||||
for ksub,vsub in v.getNodeCloud({}).iteritems():
|
||||
if not vsub.autoupdater or (branch and vsub.branch != branch):
|
||||
#break
|
||||
noAuto = True
|
||||
self.nodes_no_autoupdater[ksub] = vsub
|
||||
#else:
|
||||
if not noAuto:
|
||||
results.update(v.getNodeCloud({}))
|
||||
else:
|
||||
self.nodes_no_geo.update(v.getNodeCloud({}))
|
||||
print "Result:",len(results), region
|
||||
return results
|
||||
|
||||
|
|
|
|||
116
no_coords.py
Executable file
116
no_coords.py
Executable file
|
|
@ -0,0 +1,116 @@
|
|||
#!/usr/bin/python
|
||||
#
|
||||
# (c) 2016 descilla <mail@simon-wuellhorst.de>
|
||||
#
|
||||
# This script is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License or any later version.
|
||||
#
|
||||
# This script is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# For a copy of the GNU General Public License
|
||||
# see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import glob, os, json, collections, argparse, urllib, datetime
|
||||
from collections import OrderedDict
|
||||
|
||||
class OfflineChecker:
|
||||
def __init__(self, dataFile):
|
||||
self.printStatus = False
|
||||
self.dataSet = self.__getFile__(dataFile)
|
||||
self.fileNames = []
|
||||
self.results = {}
|
||||
self.addresses = []
|
||||
self.getAddressFiles()
|
||||
self.getFwState()
|
||||
|
||||
def __getFile__(self, nodesFile):
|
||||
if nodesFile.startswith('https://') or nodesFile.startswith('http://'):
|
||||
if self.printStatus:
|
||||
print "Download node.json from URL: " + nodesFile
|
||||
resource = urllib.urlopen(nodesFile)
|
||||
else:
|
||||
if self.printStatus:
|
||||
print "Open node.json file: " + nodesFile
|
||||
resource = open(nodesFile)
|
||||
data = json.loads(resource.read())
|
||||
resource.close()
|
||||
return data
|
||||
|
||||
def getAddressFiles(self):
|
||||
for file in glob.iglob('./nodes_adresses_*'):
|
||||
self.addresses.append(self.__getFile__(file))
|
||||
|
||||
def readFile(self):
|
||||
fil = 'operate.txt'
|
||||
results = []
|
||||
with open(fil) as lg:
|
||||
for line in lg:
|
||||
results.append(line)
|
||||
return results
|
||||
|
||||
def getNodeAddressItem(self,ipv6):
|
||||
pub = ""
|
||||
i = 0
|
||||
for adr in self.addresses:
|
||||
i +=1
|
||||
for val in adr['nodes'].itervalues():
|
||||
pub = self.getPublicAddress(val)
|
||||
if pub and pub in ipv6:
|
||||
if 'owner' in val['nodeinfo']:
|
||||
return val
|
||||
|
||||
def getNodeItem(self,ipv6):
|
||||
pub = ""
|
||||
for val in self.dataSet['nodes'].itervalues():
|
||||
pub = self.getPublicAddress(val)
|
||||
if pub and pub in ipv6:
|
||||
return val
|
||||
|
||||
def getFwState(self):
|
||||
lastDay = datetime.datetime.today() - datetime.timedelta(hours = 48)
|
||||
onlyoldernodes = False
|
||||
results = {}
|
||||
for nodeIP in self.readFile():
|
||||
nodeAddresses = self.getNodeAddressItem(nodeIP)
|
||||
node = self.getNodeItem(nodeIP)
|
||||
if node:
|
||||
nodeLastSeen = datetime.datetime.strptime(node['lastseen'],'%Y-%m-%dT%H:%M:%S')
|
||||
if nodeLastSeen < lastDay or onlyoldernodes == False:
|
||||
au = node['nodeinfo']['software']['autoupdater']['branch']
|
||||
loca = 'JA' if 'location' in node['nodeinfo'] else 'NEIN'
|
||||
if nodeAddresses:
|
||||
mail = nodeAddresses['nodeinfo']['owner']['contact'] if 'owner' in nodeAddresses['nodeinfo'] else 'NEIN'
|
||||
#mail = 'JA' if 'owner' in nodeAddresses['nodeinfo'] else 'NEIN'
|
||||
else:
|
||||
mail = 'NEIN'
|
||||
results[node['nodeinfo']['node_id']] = {
|
||||
'lastseen' : node['lastseen'],
|
||||
'ipv6' : self.getPublicAddress(node),
|
||||
'node_id' : node['nodeinfo']['node_id'],
|
||||
'name' : node['nodeinfo']['hostname'],
|
||||
'contact' : mail,
|
||||
'fw_base' : node['nodeinfo']['software']['firmware']['base'],
|
||||
'fw_release' : node['nodeinfo']['software']['firmware']['release'],
|
||||
'au_enabled' : str(node['nodeinfo']['software']['autoupdater']['enabled']),
|
||||
'au_branch' : au,
|
||||
'router_modell' : node['nodeinfo']['hardware']['model'],
|
||||
'geo' : loca,
|
||||
}
|
||||
#print node['lastseen'] + ';' + self.getPublicAddress(node) + ';' + node['nodeinfo']['node_id'] + ';' + node['nodeinfo']['hostname'] + ';' + mail + ';' + node['nodeinfo']['software']['firmware']['base'] + ';' + node['nodeinfo']['software']['firmware']['release'] + ';' + str(node['nodeinfo']['software']['autoupdater']['enabled']) + ';' + au + ';' + node['nodeinfo']['hardware']['model'] + ';' + loca
|
||||
self.printCSV(results)
|
||||
def printCSV(self, data):
|
||||
od = OrderedDict(sorted(data.items(), key=lambda x: x[1]['lastseen'], reverse=True))
|
||||
print 'zuletzt online;nodeid;Knotenname;mailaddress;Firmware Base;Firmware Release;Autoupdater;AU-Branch;Router-Modell;geo'
|
||||
for item in od.itervalues():
|
||||
print item['lastseen'] + ';' + item['node_id'] + ';' + item['name'] + ';' + item['contact'] + ';' + item['fw_base'] + ';' + item['fw_release'] + ';' + item['au_enabled'] + ';' + item['au_branch'] + ';' + item['router_modell'] + ';' + item['geo']
|
||||
def getPublicAddress(self,node):
|
||||
if 'addresses' in node['nodeinfo']['network']:
|
||||
for address in node['nodeinfo']['network']['addresses']:
|
||||
if address.startswith('2a03'):
|
||||
return address
|
||||
return None
|
||||
|
||||
dmax = OfflineChecker('https://service.freifunk-muensterland.de/maps/data/nodes.json')
|
||||
3
node.py
3
node.py
|
|
@ -4,7 +4,7 @@ from geocode import Geocode
|
|||
import time
|
||||
|
||||
class Node(object):
|
||||
def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None, autoupdater = False, branch = None):
|
||||
def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None, autoupdater = False, branch = None, isGateway = False):
|
||||
self.coder = coder
|
||||
if self.coder == None:
|
||||
self.coder = Geocode(geocoderCache = True, printStatus = True)
|
||||
|
|
@ -19,6 +19,7 @@ class Node(object):
|
|||
self.branch = branch
|
||||
self._geo = None
|
||||
self.geodata = None
|
||||
self.isGateway = isGateway
|
||||
if lat != None and lon != None:
|
||||
self.geo = {
|
||||
'lat' : lat,
|
||||
|
|
|
|||
|
|
@ -8,77 +8,90 @@ targets = {
|
|||
# {'city' : u'Münster'},
|
||||
# {'county' : u'Münster'},
|
||||
# ],
|
||||
'kreis_warendorf' : [
|
||||
{'county' : u'Kreis Warendorf'},
|
||||
],
|
||||
'kreis_coesfeld' : [
|
||||
{'county' : u'Kreis Coesfeld'},
|
||||
],
|
||||
'kreis_steinfurt_west' : [
|
||||
{'town' : u'48565'},
|
||||
{'village' : u'Wettringen'},
|
||||
{'town' : u'Ochtrup'},
|
||||
{'village' : u'Metelen'},
|
||||
{'town' : u'Horstmar'},
|
||||
{'village' : u'Laer'},
|
||||
{'village' : u'Nordwalde'},
|
||||
{'village' : u'Altenberge'},
|
||||
],
|
||||
'kreis_steinfurt_ost' : [
|
||||
{'town' : u'Emsdetten'},
|
||||
{'town' : u'Neuenkirchen'},
|
||||
{'town' : u'Rheine'},
|
||||
{'town' : u'Greven'},
|
||||
{'village' : u'Ladbergen'},
|
||||
{'town' : u'Lengerich'},
|
||||
{'town' : u'Tecklenburg'},
|
||||
{'village' : u'Lienen'},
|
||||
],
|
||||
'muenster_stadt' : [
|
||||
{'city_district' : u'Münster-Mitte'},
|
||||
{'city_district' : u'Münster-Nord'},
|
||||
{'city_district' : u'Münster-Ost'},
|
||||
{'suburb' : u'Berg Fidel'},
|
||||
{'suburb' : u'Gremmendorf'},
|
||||
{'suburb' : u'Mecklenbeck'},
|
||||
{'suburb' : u'Gievenbeck'},
|
||||
{'suburb' : u'Nienberge'},
|
||||
{'suburb' : u'Roxel'},
|
||||
{'suburb' : u'Sentruper Höhe'},
|
||||
],
|
||||
'muenster_sued' : [
|
||||
{'suburb' : u'Amelsbüren'},
|
||||
{'suburb' : u'Hiltrup'},
|
||||
{'suburb' : u'Albachten'},
|
||||
],
|
||||
'kreis_borken' : [
|
||||
{'town' : u'Ahaus'},
|
||||
{'town' : u'Bocholt'},
|
||||
{'town' : u'Borken'},
|
||||
{'town' : u'Gescher'},
|
||||
{'village' : u'Heek'},
|
||||
{'town' : u'Heiden'},
|
||||
{'town' : u'Isselburg'},
|
||||
{'village' : u'Legden'},
|
||||
{'town' : u'Raesfeld'},
|
||||
{'town' : u'Reken'},
|
||||
{'town' : u'Rhede'},
|
||||
{'village' : u'Schöppingen'},
|
||||
# 'kreis_warendorf' : [
|
||||
# {'county' : u'Kreis Warendorf'},
|
||||
# ],
|
||||
# 'kreis_coesfeld' : [
|
||||
# {'county' : u'Kreis Coesfeld'},
|
||||
# ],
|
||||
# 'kreis_steinfurt_west' : [
|
||||
# {'town' : u'48565'},
|
||||
# {'village' : u'Wettringen'},
|
||||
# {'town' : u'Ochtrup'},
|
||||
# {'village' : u'Metelen'},
|
||||
# {'town' : u'Horstmar'},
|
||||
# {'village' : u'Laer'},
|
||||
# {'village' : u'Nordwalde'},
|
||||
# {'village' : u'Altenberge'},
|
||||
# ],
|
||||
# 'kreis_steinfurt_ost' : [
|
||||
# {'town' : u'Emsdetten'},
|
||||
# {'town' : u'Neuenkirchen'},
|
||||
# {'town' : u'Rheine'},
|
||||
# {'town' : u'Greven'},
|
||||
# {'village' : u'Ladbergen'},
|
||||
# {'town' : u'Lengerich'},
|
||||
# {'town' : u'Tecklenburg'},
|
||||
# {'village' : u'Lienen'},
|
||||
# ],
|
||||
# 'muenster_stadt' : [
|
||||
# {'city_district' : u'Münster-Mitte'},
|
||||
# {'city_district' : u'Münster-Nord'},
|
||||
# {'city_district' : u'Münster-Ost'},
|
||||
# {'suburb' : u'Berg Fidel'},
|
||||
# {'suburb' : u'Gremmendorf'},
|
||||
# {'suburb' : u'Mecklenbeck'},
|
||||
# {'suburb' : u'Gievenbeck'},
|
||||
# {'suburb' : u'Nienberge'},
|
||||
# {'suburb' : u'Roxel'},
|
||||
# {'suburb' : u'Sentruper Höhe'},
|
||||
# ],
|
||||
# 'muenster_sued' : [
|
||||
# {'suburb' : u'Amelsbüren'},
|
||||
# {'suburb' : u'Hiltrup'},
|
||||
# {'suburb' : u'Albachten'},
|
||||
# ],
|
||||
# 'kreis_borken' : [
|
||||
# {'town' : u'Ahaus'},
|
||||
# {'town' : u'Bocholt'},
|
||||
# {'town' : u'Borken'},
|
||||
# {'town' : u'Gescher'},
|
||||
# {'village' : u'Heek'},
|
||||
# {'town' : u'Heiden'},
|
||||
# {'town' : u'Isselburg'},
|
||||
# {'village' : u'Legden'},
|
||||
# {'town' : u'Raesfeld'},
|
||||
# {'town' : u'Reken'},
|
||||
# {'town' : u'Rhede'},
|
||||
# {'village' : u'Schöppingen'},
|
||||
# {'town' : u'Stadtlohn'},
|
||||
# {'village' : u'Südlohn'},
|
||||
# {'town' : u'Velen'},
|
||||
# {'town' : u'Vreden'},
|
||||
# ],
|
||||
# 'sassenberg' : [
|
||||
# {'town' : u'Sassenberg'},
|
||||
# ],
|
||||
# 'telgte' : [
|
||||
# {'town' : u'Telgte'},
|
||||
# ],
|
||||
# 'warendorf_stadt' : [
|
||||
# {'town' : u'Warendorf'},
|
||||
# ]
|
||||
'stadt_stadtlohn' : [
|
||||
{'town' : u'Stadtlohn'},
|
||||
{'village' : u'Südlohn'},
|
||||
{'town' : u'Velen'},
|
||||
{'town' : u'Vreden'},
|
||||
],
|
||||
'sassenberg' : [
|
||||
{'town' : u'Sassenberg'},
|
||||
'stadt_bocholt' : [
|
||||
{'town' : u'Bocholt'},
|
||||
],
|
||||
'telgte' : [
|
||||
'stadt_telgte' : [
|
||||
{'town' : u'Telgte'},
|
||||
],
|
||||
'warendorf_stadt' : [
|
||||
'stadt_warendorf' : [
|
||||
{'town' : u'Warendorf'},
|
||||
]
|
||||
],
|
||||
}
|
||||
|
||||
#ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets)
|
||||
ds = DomainSelector(nodesFile = 'https://freifunk-muensterland.de/map/data/nodes.json', graphFile = 'https://freifunk-muensterland.de/map/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = 'stable')
|
||||
#ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None)
|
||||
ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None)
|
||||
|
|
|
|||
Loading…
Reference in a new issue