containerized the app

.gitignore

@@ -1,5 +1,4 @@
 change_file.osc*
-config.php
 lastState.txt
 lastStatisticsUpdate.txt
 log.*

Containerfile

@@ -0,0 +1,4 @@
+FROM docker.io/library/php:apache
+
+RUN docker-php-ext-install mysqli bcmath
+COPY ./www/* /var/www/html

Containerfile.data_init

@@ -0,0 +1,10 @@
+FROM docker.io/library/alpine
+
+RUN apk add mariadb-client
+COPY ./data_init/*.sql /var/sunders/init.sql
+
+CMD mariadb --host="$MYSQL_HOST" \
+  --user="$CAMERA_USER" \
+  --password="$CAMERA_USER_PASSWD" \
+  ${MYSQL_DB:+--database="$MYSQL_DB"} \
+  < "/var/sunders/init.sql"

Containerfile.utils

@@ -0,0 +1,4 @@
+FROM docker.io/library/php:cli
+
+RUN docker-php-ext-install mysqli bcmath
+COPY ./utils/* /opt/

home/sunders/init_cameras/db/createDB.sql

@@ -1,7 +1,7 @@
 create database camera;
 use camera;
 
-grant all privileges on camera.* to camera@localhost identified by 'xxxxxxxx';
+grant all privileges on camera.* to camera@localhost identified by 'asdf';
 grant select on camera.* to camselect@localhost;
 
 CREATE TABLE position (

utils/config.php

@@ -0,0 +1,15 @@
+<?php
+
+define('REPLICATE_URL', 'https://planet.openstreetmap.org/replication/minute');
+
+define('MYSQL_HOST', getenv('MYSQL_HOST'));
+define('MYSQL_DB', getenv('MYSQL_DB'));
+define('MYSQL_USER', getenv('MYSQL_USER'));
+define('MYSQL_PASSWD', getenv('MYSQL_PASSWD'));
+
+define('USE_STATISTICS', false);
+define('WEBSERVICE_COUNTRY_URL', 'http://api.geonames.org/countryCode');
+define('WEBSERVICE_USER', 'xxxxxxxx');
+define('MAX_WEBREQUESTS_PER_HOUR', 1000);
+
+?>

utils/create_db.php

@@ -0,0 +1,76 @@
+<?php
+// Admin credentials (must have privileges to create DB/users)
+$dbHost     = getenv('MYSQL_HOST');
+$dbAdmin    = getenv('MYSQL_USER');
+$dbPassword = getenv('MYSQL_PASSWD');
+$dbName = getenv('MYSQL_DB');
+
+// Variables for new users
+$cameraUser        = getenv('CAMERA_USER');
+$cameraPassword    = getenv('CAMERA_USER_PASSWD');
+
+$camSelectUser     = getenv('CAMERA_SELECT_USER');
+$camSelectPassword = getenv('CAMERA_SELECT_USER_PASSWD');
+
+// Connect to MySQL
+$conn = new mysqli($dbHost, $dbAdmin, $dbPassword);
+
+// Check connection
+if ($conn->connect_error) {
+    die("Connection failed: " . $conn->connect_error);
+}
+
+
+if (!$conn->query("CREATE DATABASE IF NOT EXISTS $dbName")) {
+    echo "Error creating database: " . $conn->error . "\n";
+}
+if (!$conn->select_db($dbName)) {
+    die("Error selecting database: " . $conn->error);
+}
+
+
+// Create camera user
+$conn->query("CREATE USER IF NOT EXISTS '$cameraUser'@'%' IDENTIFIED BY '$cameraPassword'");
+$conn->query("GRANT ALL PRIVILEGES ON $dbName.* TO '$cameraUser'@'%'");
+
+// Create camselect user
+$conn->query("CREATE USER IF NOT EXISTS '$camSelectUser'@'%' IDENTIFIED BY '$camSelectPassword'");
+$conn->query("GRANT SELECT ON $dbName.* TO '$camSelectUser'@'%'");
+
+
+// position table
+$conn->query("
+CREATE TABLE IF NOT EXISTS position (
+  id BIGINT PRIMARY KEY,
+  latitude INT,
+  longitude INT
+)
+");
+
+// tag table
+$conn->query("
+CREATE TABLE IF NOT EXISTS tag (
+  id BIGINT,
+  k VARCHAR(100),
+  v VARCHAR(10000),
+  PRIMARY KEY (id, k),
+  CONSTRAINT fk_position FOREIGN KEY (id) REFERENCES `position`(id) ON DELETE CASCADE
+)
+");
+
+$conn->query("CREATE INDEX IF NOT EXISTS LatLon ON position (latitude, longitude)");
+
+
+// sync_state table
+$conn->query("
+CREATE TABLE IF NOT EXISTS sync_state (
+  k VARCHAR(100),
+  v VARCHAR(100)
+  )
+");
+$conn->query("INSERT INTO sync_state (k, v) VALUES ('sequenceNumber', '0');");
+
+echo "Database, users, and tables created successfully.\n";
+
+$conn->close();
+?>

utils/get_sync_state.php

@@ -0,0 +1,21 @@
+<?php
+
+include "config.php";
+
+$mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWD, MYSQL_DB);
+
+if($mysqli->connect_errno) {
+  echo "Error while connecting to DB : $mysqli->error \n" ;
+  exit(1);
+}
+$mysqli->autocommit(FALSE);
+
+$result = $mysqli->query("SELECT * FROM sync_state WHERE k = 'sequenceNumber'");
+
+while($row = $result->fetch_assoc()) {
+    echo $row["v"];
+  }
+
+$mysqli->close();
+
+?>

utils/update_camera.php

@@ -0,0 +1,291 @@
+<?php
+
+include "config.php";
+
+$elementTypes = array();
+$count = 0;
+$countDelete = 0;
+$countModifyDelete = 0;
+$countModify = 0;
+$countCreate = 0;
+$mode = '';
+$curNodeAttrs = null;
+$curNodeTags = null;
+$id = 0;
+$latitude = 0;
+$longitude = 0;
+
+$mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWD, MYSQL_DB);
+
+if($mysqli->connect_errno) {
+  echo "Error while connecting to DB : $mysqli->error \n" ;
+  exit(1);
+}
+$mysqli->autocommit(FALSE);
+
+if (! ($deleteStmt = $mysqli->prepare("DELETE FROM position WHERE id=?"))) {
+  echo "Error while preparing delete position statement : " . $mysqli->error ;
+  exit(1);
+}
+$deleteStmt->bind_param('d', $id);
+
+if (! ($deleteTagStmt = $mysqli->prepare("DELETE FROM tag WHERE id=?"))) {
+  echo "Error while preparing delete tag statement : " . $mysqli->error ;
+  exit(1);
+}
+$deleteTagStmt->bind_param('d', $id);
+
+if (USE_STATISTICS) {
+  if (! ($deleteStatsStmt = $mysqli->prepare("DELETE FROM statistics WHERE id=?"))) {
+    echo "Error while preparing delete statistics statement : " . $mysqli->error ;
+    exit(1);
+  }
+  $deleteStatsStmt->bind_param('d', $id);
+}
+
+if (! ($insertStmt = $mysqli->prepare("INSERT INTO position (id, latitude, longitude) VALUES (?, ?, ?)"))) {
+  echo "Error while preparing insert position statement : " . $mysqli->error ;
+  exit(1);
+}
+$insertStmt->bind_param('dii', $id, $latitude, $longitude);
+
+if (! ($insertTagStmt = $mysqli->prepare("INSERT INTO tag (id, k, v) VALUES (?, ?, ?)"))) {
+  echo "Error while preparing insert tag statement : " . $mysqli->error ;
+  exit(1);
+}
+$insertTagStmt->bind_param('dss', $id, $k, $v);
+
+if (USE_STATISTICS) {
+  if (! ($insertStatsStmt = $mysqli->prepare("INSERT INTO statistics (id, ts, version) VALUES (?, ?, ?)"))) {
+    echo "Error while preparing insert statistics statement : " . $mysqli->error ;
+    exit(1);
+  }
+  $insertStatsStmt->bind_param('dsi', $id, $ts, $version);
+}
+
+function printDebug() {
+  global $elementTypes, $count, $countDelete, $countModifyDelete, $countModify, $countCreate;
+
+  echo "== $count ==============================\n";
+  foreach($elementTypes as $k => $v) {
+    echo "$k : $v\n";
+  }
+  echo "++++++ Surveillance nodes : ++++++\n";
+  echo "$countDelete deletions\n";
+  echo "$countModifyDelete modifications --> deletions\n";
+  echo "$countModify modifications\n";
+  echo "$countCreate creations\n";
+}
+
+function printDebugCurNode() {
+  global $curNodeAttrs, $curNodeTags, $mode;
+
+  echo "=============\n";
+  echo "$mode : " . $curNodeAttrs['id'] ." (". $curNodeAttrs['lat'] . " x " . $curNodeAttrs['lon'] . ") :  " . $curNodeAttrs['user'] . "\n";
+  if (! empty($curNodeAttrs)) {
+    echo "  => Attributes :\n";
+    foreach($curNodeAttrs as $k => $v) {
+      echo "   $k : $v\n";
+    }
+  } else {
+     echo "  => No attributes\n";
+  }
+  if (! empty($curNodeTags)) {
+    echo "  => Tags :\n";
+    foreach($curNodeTags as $k => $v) {
+      echo "   $k : $v\n";
+    }
+  } else {
+     echo "  => No tags\n";
+  }
+}
+
+function startElement ($parser, $name, $attrs) {
+  global $elementTypes, $count, $mode, $curNodeAttrs, $curNodeTags;
+  $count++;
+
+  if (array_key_exists($name, $elementTypes)) {
+    $elementTypes[$name] += 1;
+  } else {
+    $elementTypes[$name] = 1;
+  }
+
+  if ($name == 'modify' || $name == 'delete' or $name == 'create') {
+    $mode = $name;
+
+  } else if ($name == 'node') {
+    $curNodeAttrs = $attrs;
+    $curNodeTags = array();
+
+  } else if ($name == 'tag') {
+    $curNodeTags[$attrs['k']] = $attrs['v'];
+  }
+}
+
+function endElement ($parser, $name) {
+  if (USE_STATISTICS) {
+    global $mode, $deleteStmt, $deleteTagStmt, $deleteStatsStmt, $insertStmt, $insertTagStmt, $insertStatsStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $ts, $version, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate;
+  } else {
+    global $mode, $deleteStmt, $deleteTagStmt, $insertStmt, $insertTagStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate;
+  }
+
+  if ($name == 'modify' || $name == 'delete' or $name == 'create') {
+    $mode = '';
+  } else if ($name == 'node') {
+    if ($mode == 'delete') {
+      $id = $curNodeAttrs['id'];
+
+      if (USE_STATISTICS) {
+        if (! $deleteStatsStmt->execute()) {
+          echo "***** Error : Deleting statistics $id : ". $deleteStatsStmt->error . "\n";
+        }
+      }
+
+      if (! $deleteTagStmt->execute()) {
+        echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n";
+      }
+      if (! $deleteStmt->execute()) {
+        echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n";
+      }
+      if ($deleteStmt->affected_rows > 0) {
+        $countDelete++;
+        $mysqli->commit();
+        printDebugCurNode();
+      }
+
+    } else if ($mode == 'modify' || $mode == 'create') {
+      if (! empty($curNodeTags)
+          && array_key_exists('man_made', $curNodeTags)
+          && $curNodeTags['man_made'] == 'surveillance') {
+
+        printDebugCurNode();
+
+        $id = $curNodeAttrs['id'];
+
+        if ($mode == 'modify') {
+          $countModify++;
+
+          if (USE_STATISTICS) {
+            if (! $deleteStatsStmt->execute()) {
+              echo "***** Error : Deleting statistics $id for modification : ". $deleteStatsStmt->error . "\n";
+            }
+          }
+
+          if (! $deleteTagStmt->execute()) {
+            echo "***** Error : Deleting tags $id for modification : ". $deleteTagStmt->error . "\n";
+          }
+          if (! $deleteStmt->execute()) {
+            echo "***** Error : Deleting $id for modification : ". $deleteStmt->error . "\n";
+          }
+        } else {
+          $countCreate++;
+        }
+
+        $latitude = (int) ($curNodeAttrs['lat'] * 10000000);
+        $longitude = (int) ($curNodeAttrs['lon'] * 10000000);
+
+        if (! $insertStmt->execute()) {
+          echo "***** Error : inserting $id ($latitude x $longitude) : ". $insertStmt->error . "\n";
+        }
+
+        $k = 'lat';
+        $v = $curNodeAttrs['lat'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting latitude $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        $k = 'lon';
+        $v = $curNodeAttrs['lon'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting longitude $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        $k = 'userid';
+        $v = $curNodeAttrs['user'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting user $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        $k = 'version';
+        $v = $curNodeAttrs['version'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting version $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        $k = 'timestamp';
+        $v = $curNodeAttrs['timestamp'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting timestamp $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        foreach($curNodeTags as $k => $v) {
+          if (! $insertTagStmt->execute()) {
+            echo "***** Error : inserting tag $k => $v for $id : ". $insertTagStmt->error . "\n";
+          }
+        }
+
+        if (USE_STATISTICS) {
+          $ts = $curNodeAttrs['timestamp'];
+          $version = $curNodeAttrs['version'];
+          if (! $insertStatsStmt->execute()) {
+            echo "***** Error : inserting ts $ts, version $version for $id : ". $insertStatsStmt->error . "\n";
+          }
+        }
+
+        $mysqli->commit();
+      } else if ($mode == 'modify') {
+        // delete former surveillance nodes that were modified to non-surveillance nodes
+        $id = $curNodeAttrs['id'];
+
+        if (USE_STATISTICS) {
+          if (! $deleteStatsStmt->execute()) {
+            echo "***** Error : Deleting statistics $id for non-surveillance node : ". $deleteStatsStmt->error . "\n";
+          }
+        }
+        if (! $deleteTagStmt->execute()) {
+          echo "***** Error : Deleting tags $id for non-surveillance node : ". $deleteTagStmt->error . "\n";
+        }
+        if (! $deleteStmt->execute()) {
+          echo "***** Error : Deleting $id for non-surveillance node : ". $deleteStmt->error . "\n";
+        }
+
+        if ($deleteStmt->affected_rows > 0) {
+          $countModifyDelete++;
+          $mysqli->commit();
+          printDebugCurNode();
+          echo "  ==> delete because modified to non-surveillance node\n";
+        }
+      }
+    }
+
+    $curNodeAttrs = null;
+    $curNodeTags = null;
+  }
+}
+
+$file="change_file.osc";
+
+$xml_parser = xml_parser_create();
+xml_parser_set_option($xml_parser, XML_OPTION_CASE_FOLDING, false);
+xml_set_element_handler($xml_parser, "startElement", "endElement");
+
+if (!($fp = fopen($file, "r"))) {
+  die("could not open XML input");
+}
+
+while ($data = fread($fp, 4096)) {
+  if (!xml_parse($xml_parser, $data, feof($fp))) {
+    die(sprintf("XML error: %s at line %d",
+                xml_error_string(xml_get_error_code($xml_parser)),
+                xml_get_current_line_number($xml_parser)));
+  }
+}
+
+$sql="UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';";
+$mysqli->query($sql);
+$mysqli->commit();
+
+printDebug();
+$mysqli->close();
+
+?>

utils/update_camera.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+exedir=$(cd `dirname "$0"`; pwd)
+
+REPLICATE_URL=`grep "REPLICATE_URL" "$exedir/config.php"  | sed -e 's/.*http/http/' -e 's#...$##' `
+
+if [ -e "/var/lock/update_camera" ]
+then
+  # Maybe an other update is running
+  otherPid=$(cat "/var/lock/update_camera")
+  count=$(ps $otherPid | grep -c `basename "$0"`)
+
+  if [ $count -gt 0 ]
+  then
+    echo "$0 is running yet. Exiting." >&2
+    exit 1
+  fi
+fi
+
+# OK. We can update the database...
+echo $$ >  "/var/lock/update_camera"
+
+exeDir=$(cd `dirname "$0"`; pwd);
+cd "$exeDir"
+
+if [ -e "state.txt" ]
+then
+  rm "state.txt"
+fi
+
+#TODO get state from db
+# Read the last update timestamp
+# lastTimestamp=$(grep "^timestamp=" "lastState.txt" | cut -d'=' -f2-)
+# lastSeqNum=$(grep "^sequenceNumber=" "lastState.txt" | cut -d'=' -f2-)
+
+lastSeqNum=$(php get_sync_state.php)
+
+curl -L "$REPLICATE_URL/state.txt" -o state.txt 
+
+newTimestamp=$(grep "^timestamp=" "state.txt" | cut -d'=' -f2-)
+newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-)
+
+if [ $newSeqNum -eq $lastSeqNum ]
+then
+  echo "No new file to be processed"
+  exit 1
+fi
+mkdir logs
+curSeqNum=$lastSeqNum
+while [ $curSeqNum -lt $newSeqNum ]
+do
+  curSeqNum=$(( $curSeqNum + 1 ))
+
+  logFileName="logs/log.$curSeqNum"
+
+  if [ -e "change_file.osc" ]
+  then
+    rm "change_file.osc"
+  fi
+
+  if [ -e "change_file.osc.gz" ]
+  then
+    rm "change_file.osc.gz"
+  fi
+
+  echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName"
+
+  targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'`
+  targetDirName="$REPLICATE_URL/$targetDirName"
+
+  curl -L "$targetDirName" -o change_file.osc.gz 2>&1 | tee -a $logFileName
+  if [ $? -gt 0 ]
+  then
+    echo "Error during recovery of $targetDirName" | tee -a $logFileName
+    exit 1
+  fi
+
+  gunzip change_file.osc.gz 2>&1 | tee -a $logFileName
+  if [ $? -gt 0 ]
+  then
+    echo "Error during decompression of $targetDirName" | tee -a $logFileName
+    exit 1
+  fi
+
+  php update_camera.php $curSeqNum 2>&1 | tee -a $logFileName
+
+  targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'`
+
+  curl -L "$targetDirName" -o lastState.txt 2>&1 | tee -a $logFileName
+  echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName"
+done
+
+rm "/var/lock/update_camera"

www/sunders/add-lists.php

@@ -1,4 +1,5 @@
 <?php
+  error_reporting(0);
   // Convert the content of the symbology JSON file to HTML.
   function addListSymbology($jsonPath, $i18n, $i18nDefault) {
     global $pathToWebFolder;

www/sunders/camera.php

@@ -1,7 +1,7 @@
 <?php
-
+  error_reporting(0);
   include $pathToWebFolder.'config.php';
-
+  
   define('MAX_POINTS_FOR_QUICKHULL', 3000);
 
   class OsmPoint {

www/sunders/config.php

@@ -0,0 +1,19 @@
+<?php
+
+define('DEFAULT_ZOOM', 15);
+define('DEFAULT_LAT', 52.37672572);
+define('DEFAULT_LON', 9.73787785);
+define('DEFAULT_LANGUAGE', 'en');
+define('DEFAULT_PIE', 'country');
+define('DEFAULT_TIME', 'single');
+
+define('MYSQL_HOST', getenv('MYSQL_HOST'));
+define('MYSQL_DB', getenv('MYSQL_DB'));
+define('MYSQL_USER', getenv('MYSQL_USER'));
+define('MYSQL_PASSWD', getenv('MYSQL_PASSWD'));
+
+
+
+define('USE_STATISTICS', false);
+
+?>