- Added support of GW-mesh-links (Domaene 14)

- Added output of nodes without geo or disabled autoupdater
- Changed output styling
- Several minor changes

domain_selector.py

@@ -22,6 +22,8 @@ class DomainSelector:
 				self.writeConfigFiles(nodes,k)
 				self.writeDumpFile(nodes,k)
 				nodes = {}
+			self.writeConfigFiles(self.graph.nodes_no_autoupdater,"no_autoupdater")
+			self.writeConfigFiles(self.graph.nodes_no_geo,"no_geo")
 
 	def __getFile__(self, nodesFile):
 		if nodesFile.startswith('https://') or nodesFile.startswith('http://'):
@@ -39,14 +41,12 @@ class DomainSelector:
 	def writeConfigFiles(self, nodes, name):
 		maxDepth = self.maxDepth(nodes)
 		for i in range(0,maxDepth):
-			content = 'geo $switch {\n\tdefault\t0;'
+			content = 'geo $switch {\n  default 0;'
 			f = open(self.dataPath+'/'+name+'_node_level'+str(i),'w')
 			for node in nodes.itervalues():
 				if node.stepsToVpn == i:
 					if node.ipv6 and node.hostname:
-						content += '\n\t'+node.ipv6+'\t1;\t #'+node.hostname
+						content += '\n  '+node.ipv6+' 1; #'+node.hostname
-					#else:
-					#	print node.nodeid
 			content += '\n}'
 			f.write(content.encode('utf8'))
 			f.close()
@@ -69,4 +69,4 @@ class DomainSelector:
 		for v in nodes.itervalues():
 			if v.stepsToVpn > maxDepth:
 				maxDepth = v.stepsToVpn
-		return maxDepth+1
+		return maxDepth+1

graph.py

@@ -13,6 +13,8 @@ class Graph:
 		self.data = graphData
 		self.nodes = nodesData
 		self.nodes_list = {}
+		self.nodes_no_autoupdater = {}
+		self.nodes_no_geo = {}
 		self.parseNodes()
 		self.parseLinks()
 		self.calculateStepsToVpn()
@@ -21,14 +23,17 @@ class Graph:
 	def parseNodes(self):
 		for k,v in self.nodes['nodes'].iteritems():
 			lat, lon = self.getGeo(k)
-			node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder, autoupdater = self.getAutoupdaterStatus(k), branch = self.getBranch(k))
+			node = Node(k, ipv6 = self.getPublicAddress(k), hostname = self.getHostname(k), isOnline = self.getOnlineState(k), lat=lat, lon=lon, coder = self.coder, autoupdater = self.getAutoupdaterStatus(k), branch = self.getBranch(k), isGateway = self.getIsGateway(k))
 			self.nodes_list[k] = node
 
 	def parseLinks(self):
 		link_nodes = self.data['batadv']['nodes']
 		for link in self.data['batadv']['links']:
 			if 'node_id' in link_nodes[link['source']].keys() and 'node_id' in link_nodes[link['target']].keys():#else it is a vpn link
-				self.setLinkBetween(link_nodes[link['source']]['node_id'], link_nodes[link['target']]['node_id'])
+				if self.nodes_list[link_nodes[link['source']]['node_id']].isGateway == True or self.nodes_list[link_nodes[link['target']]['node_id']].isGateway:
+					self.setVpnLink(link['source'], link['target'])
+				else:
+					self.setLinkBetween(link_nodes[link['source']]['node_id'], link_nodes[link['target']]['node_id'])
 			else:
 				self.setVpnLink(link['source'], link['target'])
 					
@@ -46,11 +51,11 @@ class Graph:
 			}
 
 	def setVpnLink(self, src, dst):
-		if 'node_id' not in self.data['batadv']['nodes'][src].keys():
+		if 'node_id' not in self.data['batadv']['nodes'][src].keys() or self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].isGateway == True:
-			if self.data['batadv']['nodes'][dst]['node_id']:
+			if 'node_id' in self.data['batadv']['nodes'][dst]:
 				self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].stepsToVpn = 0
-		elif 'node_id' not in self.data['batadv']['nodes'][dst].keys():
+		elif 'node_id' not in self.data['batadv']['nodes'][dst].keys() or self.nodes_list[self.data['batadv']['nodes'][dst]['node_id']].isGateway == True:
-			if self.data['batadv']['nodes'][src]['node_id']:
+			if 'node_id' in self.data['batadv']['nodes'][src]:
 				self.nodes_list[self.data['batadv']['nodes'][src]['node_id']].stepsToVpn = 0
 
 	def calculateStepsToVpn(self):
@@ -73,6 +78,9 @@ class Graph:
 	def getHostname(self,node_id):
 		return self.nodes['nodes'][node_id]['nodeinfo']['hostname']
 
+	def getIsGateway(self,node_id):
+		return self.nodes['nodes'][node_id]['flags']['gateway']
+
 	def getAutoupdaterStatus(self, node_id):
 		#return True
 		if 'autoupdater' in self.nodes['nodes'][node_id]['nodeinfo']['software']:
@@ -108,14 +116,22 @@ class Graph:
 
 	def getNodeCloudsIn(self, region, branch = 'stable'):
 		results = {}
+		noAuto = False
 		for k,v in self.getAllLevelXNodes(0).iteritems():
-			if v.geodata != None and v.isOnline == True:
+			if v.isOnline == True:
-				if v.isInRegion(region):
+				if v.geodata != None:
-					for ksub,vsub in v.getNodeCloud({}).iteritems():
+					if v.isInRegion(region):
-						if not vsub.autoupdater or vsub.branch != branch:
+						noAuto = False
-							break
+						for ksub,vsub in v.getNodeCloud({}).iteritems():
-					else:
+							if not vsub.autoupdater or (branch and vsub.branch != branch):
-						results.update(v.getNodeCloud({}))
+								#break
+								noAuto = True
+								self.nodes_no_autoupdater[ksub] = vsub
+						#else:
+						if not noAuto:
+							results.update(v.getNodeCloud({}))
+				else:
+					self.nodes_no_geo.update(v.getNodeCloud({}))
 		print "Result:",len(results), region
 		return results
 

no_coords.py

@@ -0,0 +1,116 @@
+#!/usr/bin/python
+#
+# (c) 2016 descilla <mail@simon-wuellhorst.de>
+#
+# This script is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License or any later version.
+#
+# This script is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY.  See the
+# GNU General Public License for more details.
+#
+# For a copy of the GNU General Public License
+# see <http://www.gnu.org/licenses/>.
+#
+import glob, os, json, collections, argparse, urllib, datetime
+from collections import OrderedDict
+
+class OfflineChecker:
+	def __init__(self, dataFile):
+		self.printStatus = False
+		self.dataSet = self.__getFile__(dataFile)
+		self.fileNames = []
+		self.results = {}
+		self.addresses = []
+		self.getAddressFiles()
+		self.getFwState()
+		
+	def __getFile__(self, nodesFile):
+		if nodesFile.startswith('https://') or nodesFile.startswith('http://'):
+			if self.printStatus:
+				print "Download node.json from URL: " + nodesFile
+			resource = urllib.urlopen(nodesFile)
+		else:
+			if self.printStatus:
+				print "Open node.json file: " + nodesFile
+			resource = open(nodesFile)
+		data = json.loads(resource.read())
+		resource.close()
+		return data
+
+	def getAddressFiles(self):
+		for file in glob.iglob('./nodes_adresses_*'):
+			self.addresses.append(self.__getFile__(file))
+
+	def readFile(self):
+		fil = 'operate.txt'
+		results = []
+		with open(fil) as lg:
+			for line in lg:
+				results.append(line)
+		return results
+
+	def getNodeAddressItem(self,ipv6):
+		pub = ""
+		i = 0
+		for adr in self.addresses:
+			i +=1
+			for val in adr['nodes'].itervalues():
+				pub = self.getPublicAddress(val)
+				if pub and pub in ipv6:
+					if 'owner' in val['nodeinfo']:
+						return val
+
+	def getNodeItem(self,ipv6):
+		pub = ""
+		for val in self.dataSet['nodes'].itervalues():
+			pub = self.getPublicAddress(val)
+			if pub and pub in ipv6:
+				return val
+
+	def getFwState(self):
+		lastDay = datetime.datetime.today() - datetime.timedelta(hours = 48)
+		onlyoldernodes = False
+		results = {}
+		for nodeIP in self.readFile():
+			nodeAddresses = self.getNodeAddressItem(nodeIP)
+			node = self.getNodeItem(nodeIP)
+			if node:
+				nodeLastSeen = datetime.datetime.strptime(node['lastseen'],'%Y-%m-%dT%H:%M:%S')
+				if nodeLastSeen < lastDay or onlyoldernodes == False:
+					au = node['nodeinfo']['software']['autoupdater']['branch']
+					loca = 'JA' if 'location' in node['nodeinfo'] else 'NEIN'
+					if nodeAddresses:
+						mail = nodeAddresses['nodeinfo']['owner']['contact'] if 'owner' in nodeAddresses['nodeinfo'] else 'NEIN'
+						#mail = 'JA' if 'owner' in nodeAddresses['nodeinfo'] else 'NEIN'
+					else:
+						mail = 'NEIN'
+					results[node['nodeinfo']['node_id']] = {
+						'lastseen' : node['lastseen'],
+						'ipv6' : self.getPublicAddress(node),
+						'node_id' : node['nodeinfo']['node_id'],
+						'name' : node['nodeinfo']['hostname'],
+						'contact' : mail,
+						'fw_base' : node['nodeinfo']['software']['firmware']['base'],
+						'fw_release' : node['nodeinfo']['software']['firmware']['release'],
+						'au_enabled' : str(node['nodeinfo']['software']['autoupdater']['enabled']),
+						'au_branch' : au,
+						'router_modell' : node['nodeinfo']['hardware']['model'],
+						'geo' : loca,
+					}
+					#print node['lastseen'] + ';' + self.getPublicAddress(node) + ';' + node['nodeinfo']['node_id'] + ';' + node['nodeinfo']['hostname'] + ';' + mail + ';' + node['nodeinfo']['software']['firmware']['base'] + ';' + node['nodeinfo']['software']['firmware']['release'] + ';' + str(node['nodeinfo']['software']['autoupdater']['enabled']) + ';' + au + ';' + node['nodeinfo']['hardware']['model'] + ';' + loca
+		self.printCSV(results)
+	def printCSV(self, data):
+		od = OrderedDict(sorted(data.items(), key=lambda x: x[1]['lastseen'], reverse=True))
+		print 'zuletzt online;nodeid;Knotenname;mailaddress;Firmware Base;Firmware Release;Autoupdater;AU-Branch;Router-Modell;geo'
+		for item in od.itervalues():
+			print item['lastseen'] + ';' + item['node_id'] + ';' + item['name'] + ';' + item['contact'] + ';' + item['fw_base'] + ';' + item['fw_release'] + ';' + item['au_enabled'] + ';' + item['au_branch'] + ';' + item['router_modell'] + ';' + item['geo']
+	def getPublicAddress(self,node):
+		if 'addresses' in node['nodeinfo']['network']:
+			for address in node['nodeinfo']['network']['addresses']:
+				if address.startswith('2a03'):
+					return address
+		return None
+
+dmax = OfflineChecker('https://service.freifunk-muensterland.de/maps/data/nodes.json')

node.py

@@ -4,7 +4,7 @@ from geocode import Geocode
 import time
 
 class Node(object):
-	def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None, autoupdater = False, branch = None):
+	def __init__(self, nodeid, ipv6 = None, hostname = None, isOnline = False, lastSeen = None, lat = None, lon = None, coder = None, autoupdater = False, branch = None, isGateway = False):
 		self.coder = coder
 		if self.coder == None:
 			self.coder = Geocode(geocoderCache = True, printStatus = True)
@@ -19,6 +19,7 @@ class Node(object):
 		self.branch = branch
 		self._geo = None
 		self.geodata = None
+		self.isGateway = isGateway
 		if lat != None and lon != None:
 			self.geo = {
 				'lat' : lat,

node_hierarchy.py

@@ -8,77 +8,90 @@ targets = {
 #		{'city' : u'Münster'}, 
 #		{'county' : u'Münster'},
 #	],
-	'kreis_warendorf' : [
+	# 'kreis_warendorf' : [
-		{'county' : u'Kreis Warendorf'},
+	# 	{'county' : u'Kreis Warendorf'},
-	],
+	# ],
-	'kreis_coesfeld' : [
+	# 'kreis_coesfeld' : [
-		{'county' : u'Kreis Coesfeld'},
+	# 	{'county' : u'Kreis Coesfeld'},
-	],
+	# ],
-	'kreis_steinfurt_west' : [
+	# 'kreis_steinfurt_west' : [
-		{'town' : u'48565'},
+	# 	{'town' : u'48565'},
-		{'village' : u'Wettringen'},
+	# 	{'village' : u'Wettringen'},
-		{'town' : u'Ochtrup'},
+	# 	{'town' : u'Ochtrup'},
-		{'village' : u'Metelen'},
+	# 	{'village' : u'Metelen'},
-		{'town' : u'Horstmar'},
+	# 	{'town' : u'Horstmar'},
-		{'village' : u'Laer'},
+	# 	{'village' : u'Laer'},
-		{'village' : u'Nordwalde'},
+	# 	{'village' : u'Nordwalde'},
-		{'village' : u'Altenberge'},
+	# 	{'village' : u'Altenberge'},
-	],
+	# ],
-	'kreis_steinfurt_ost' : [
+	# 'kreis_steinfurt_ost' : [
-		{'town' : u'Emsdetten'},
+	# 	{'town' : u'Emsdetten'},
-		{'town' : u'Neuenkirchen'},
+	# 	{'town' : u'Neuenkirchen'},
-		{'town' : u'Rheine'},
+	# 	{'town' : u'Rheine'},
-		{'town' : u'Greven'},
+	# 	{'town' : u'Greven'},
-		{'village' : u'Ladbergen'},
+	# 	{'village' : u'Ladbergen'},
-		{'town' : u'Lengerich'},
+	# 	{'town' : u'Lengerich'},
-		{'town' : u'Tecklenburg'},
+	# 	{'town' : u'Tecklenburg'},
-		{'village' : u'Lienen'},
+	# 	{'village' : u'Lienen'},
-	],
+	# ],
-	'muenster_stadt' : [
+	# 'muenster_stadt' : [
-		{'city_district' : u'Münster-Mitte'},
+	# 	{'city_district' : u'Münster-Mitte'},
-		{'city_district' : u'Münster-Nord'},
+	# 	{'city_district' : u'Münster-Nord'},
-		{'city_district' : u'Münster-Ost'},
+	# 	{'city_district' : u'Münster-Ost'},
-		{'suburb' : u'Berg Fidel'},
+	# 	{'suburb' : u'Berg Fidel'},
-		{'suburb' : u'Gremmendorf'},
+	# 	{'suburb' : u'Gremmendorf'},
-		{'suburb' : u'Mecklenbeck'},
+	# 	{'suburb' : u'Mecklenbeck'},
-		{'suburb' : u'Gievenbeck'},
+	# 	{'suburb' : u'Gievenbeck'},
-		{'suburb' : u'Nienberge'},
+	# 	{'suburb' : u'Nienberge'},
-		{'suburb' : u'Roxel'},
+	# 	{'suburb' : u'Roxel'},
-		{'suburb' : u'Sentruper Höhe'},
+	# 	{'suburb' : u'Sentruper Höhe'},
-	],
+	# ],
-	'muenster_sued' : [
+	# 'muenster_sued' : [
-		{'suburb' : u'Amelsbüren'},
+	# 	{'suburb' : u'Amelsbüren'},
-		{'suburb' : u'Hiltrup'},
+	# 	{'suburb' : u'Hiltrup'},
-		{'suburb' : u'Albachten'},
+	# 	{'suburb' : u'Albachten'},
-	],
+	# ],
-	'kreis_borken' : [
+	# 'kreis_borken' : [
-		{'town' : u'Ahaus'},
+	# 	{'town' : u'Ahaus'},
-		{'town' : u'Bocholt'},
+	# 	{'town' : u'Bocholt'},
-		{'town' : u'Borken'},
+	# 	{'town' : u'Borken'},
-		{'town' : u'Gescher'},
+	# 	{'town' : u'Gescher'},
-		{'village' : u'Heek'},
+	# 	{'village' : u'Heek'},
-		{'town' : u'Heiden'},
+	# 	{'town' : u'Heiden'},
-		{'town' : u'Isselburg'},
+	# 	{'town' : u'Isselburg'},
-		{'village' : u'Legden'},
+	# 	{'village' : u'Legden'},
-		{'town' : u'Raesfeld'},
+	# 	{'town' : u'Raesfeld'},
-		{'town' : u'Reken'},
+	# 	{'town' : u'Reken'},
-		{'town' : u'Rhede'},
+	# 	{'town' : u'Rhede'},
-		{'village' : u'Schöppingen'},
+	# 	{'village' : u'Schöppingen'},
+	# 	{'town' : u'Stadtlohn'},
+	# 	{'village' : u'Südlohn'},
+	# 	{'town' : u'Velen'},
+	# 	{'town' : u'Vreden'},
+	# ],
+	# 'sassenberg' : [
+	# 	{'town' : u'Sassenberg'},
+	# ],
+	# 'telgte' : [
+	# 	{'town' : u'Telgte'},
+	# ],
+	# 'warendorf_stadt' : [
+	# 	{'town' : u'Warendorf'},
+	# ]
+	'stadt_stadtlohn' : [
 		{'town' : u'Stadtlohn'},
-		{'village' : u'Südlohn'},
-		{'town' : u'Velen'},
-		{'town' : u'Vreden'},
 	],
-	'sassenberg' : [
+	'stadt_bocholt' : [
-		{'town' : u'Sassenberg'},
+		{'town' : u'Bocholt'},
 	],
-	'telgte' : [
+	'stadt_telgte' : [
 		{'town' : u'Telgte'},
 	],
-	'warendorf_stadt' : [
+	'stadt_warendorf' : [
 		{'town' : u'Warendorf'},
-	]
+	],
 }
 
 #ds = DomainSelector(nodesFile = 'nodes.json', graphFile = 'graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets)
-ds = DomainSelector(nodesFile = 'https://freifunk-muensterland.de/map/data/nodes.json', graphFile = 'https://freifunk-muensterland.de/map/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = 'stable')
+#ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data_legacy/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None)
+ds = DomainSelector(nodesFile = 'https://service.freifunk-muensterland.de/maps/data/nodes.json', graphFile = 'https://service.freifunk-muensterland.de/maps/data/graph.json', printStatus = True, dataPath = '../domaenensplit_webserver_config/', targets = targets, branch = None)