Merge pull request 'Containerizes application' (#12) from container into main

Reviewed-on: #12

.forgejo/workflows/images.yml

@@ -0,0 +1,57 @@
+name: Build (and tag) Images
+
+on:
+  push:
+    branches:
+    workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.hamburg.ccc.de
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+      
+      - name: Set image tags
+        id: vars
+        run: |
+          if [ "${{ github.ref_name }}" = "main" ] && [ "${{ github.event_name }}" = "push" ]; then
+            echo "tag=latest" >> $GITHUB_OUTPUT
+          else
+            echo "tag=${{ github.ref_name }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build web image
+        run: |
+          docker build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} .
+        working-directory: ./web
+
+      - name: Build data_handler image
+        run: |
+          docker build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} .
+        working-directory: ./data_handler
+
+      - name: Push images to Container Registry
+        run: |
+          docker push git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }}
+          docker push git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }}
+        if: github.event_name == 'push'
+
+      - name: Update docker-compose.yml image tags
+        run: |
+          sed -i "s/:latest/:${{ steps.vars.outputs.tag }}/g" docker-compose.yml
+
+      - name: Start Docker Compose services
+        run: |
+          docker compose up -d --wait
+          docker compose down 

.gitignore

@@ -1,6 +1,6 @@
 change_file.osc*
-config.php
 lastState.txt
 lastStatisticsUpdate.txt
 log.*
 state.txt
+migrations

LICENSEINFO.md

@@ -1,7 +1,7 @@
 **Surveillance under Surveillance** uses code, data, icons and fonts that are licensed under different licenses:
 
-* Code of [osmcamera](https://github.com/khris78/osmcamera) — [CC-BY-SA / MIT / GPLv3 / WTFPL](./www/sunders/files/license_osmcamera.txt)
-* Code of [Leaflet/Leaflet](https://github.com/Leaflet/Leaflet) — [BSD-2-Clause](./www/sunders/files/license_Leaflet.txt)
+* Code of [osmcamera](https://github.com/khris78/osmcamera) — [CC-BY-SA / MIT / GPLv3 / WTFPL](./web/www/sunders/files/license_osmcamera.txt)
+* Code of [Leaflet/Leaflet](https://github.com/Leaflet/Leaflet) — [BSD-2-Clause](./web/www/sunders/files/license_Leaflet.txt)
 * Data of [OpenStreetMap](https://www.openstreetmap.org/) — [ODbL / CC BY-SA](https://www.openstreetmap.org/copyright)
 * Data of [OpenStreetMap Nominatim](https://nominatim.openstreetmap.org/) — [ODbL](https://www.openstreetmap.org/copyright)
 * Data of [GeoNames](https://www.geonames.org/) — [CC BY 3.0](https://creativecommons.org/licenses/by/3.0/)

README.md

@@ -22,124 +22,35 @@ Color | Description
 
 A running instance of this project can be visited at [https://sunders.uber.space/](https://sunders.uber.space/).
 
+## Prerequisites
+
+- [Docker](https://www.docker.com)
+
 ## Installation
 
-If you like to run Surveillance under Surveillance on your own LAMP or LNMP server follow these steps:
+To run Surveillance using docker, decide between development/testing or production deployments:
 
-1. Get the sources
+### Development/testing
 
- - Copy the content of **home/sunders/** to your home directory, e.g. to **~/sunders/**.
+1. Clone this repository
 
- - Copy the content of **www/sunders/** to your server's www directory, e.g. to **/var/www/sunders/**.
+2. Run `docker compose up` and wait for services to start up
 
-2. Set up the database
+3. Visit `http://localhost:8080/` in a browser; you should see an interactive OpenStreetMap
 
- - Change to the directory **~/sunders/init_cameras/db/**.
+_tbd.: Populate camera data_
 
- - Open the file **createDB.sql** and enter a password for the new database user **camera**.
+### Production
 
- - Create the database **camera** by executing the file **createDB.sql**.
+1. Close this repository
 
-    `mysql -h localhost -u root --password=[mysql root password] < createDB.sql`
+2. Hardern security by opening `docker-compose.yml` and modifying `MYSQL_PASSWORD` and `MYSQL_ROOT_PASSWORD`
 
-3. Initialize the database
+3. Run `docker compose up` and wait for services to start up
 
-  Decide whether you like to start with the surveillance entries of the first planet.osm file from September 12, 2012 or if you like to start with the latest planet.osm file or an extract of an individual country or region.
+3. Visit `http://localhost:8080/` in a browser to open the web interface
 
-  **Start with the surveillance entries of the first planet.osm file from September 12, 2012**
-
-  Pros: You don't have to download the latest +50GB planet.osm file to create a sql import file.
-
-  Cons: It could take several days until your database is up-to-date and contains all surveillance entries that have been added between September 12, 2012 and today. Furthermore you start with a data record for the whole planet. Maybe you are only interested in the data of a certain country or region.
-
-  - Execute the file **initializeDB_planet_20120912.sql** for database user **camera**.
-
-    `mysql camera -h localhost -u camera --password=[camera user password] < initializeDB_planet_20120912.sql`
-
-  **Start with the latest planet.osm file or an extract of an individual country or state**
-
-  Pros: You start with the latest data records. Furthermore you can choose what country or region you like to map.
-
-  Cons: If you like to map the whole planet you have to download the latest +50GB planet.osm file. According to your internet connection this could take a while. At last you have to install the command line Openstreetmap data processor [Osmosis](https://wiki.openstreetmap.org/wiki/Osmosis) on your computer.
-
-  - Download the latest osm.bz2 file you like to extract the data from, e.g. from [planet.openstreetmap.org](https://planet.openstreetmap.org/) or from [download.geofabrik.de](http://download.geofabrik.de/). They also offer a MD5 sum to verify the downloaded file.
-
-  - Copy the just downloaded osm.bz2 file to the directory **~/sunders/init_cameras/**.
-
-  - Open the file **~/sunders/init_cameras/createInitialDataFiles.sh** and enter the name of the osm.bz2 file as **XML_FILE**.
-
-    `XML_FILE=[file name].osm.bz2`
-
-  - Execute **createInitialDataFiles.sh** to create the files **surveillance.osm** and **initializeDB.sql**.
-
-  - Move the new files **surveillance.osm** and **initializeDB.sql** to the directory **~/sunders/init_cameras/db/** and change to that directory.
-
-  - Execute the file **initializeDB.sql** for database user **camera**.
-
-    `mysql camera -h localhost -u camera --password=[camera user password] < initializeDB.sql`
-
-4. Update the database
-
-  - Change to the directory **~/sunders/update_cameras/**.
-
-  - Rename the file **config.php.example** to **config.php**.
-
-  - Open the file **config.php** and enter the **MYSQL_PASSWD** of the database user **camera**. Furthermore enter the **REPLICATE_URL** that fits to your project, e.g. from [planet.openstreetmap.org](https://planet.openstreetmap.org/replication/) or from [download.openstreetmap.fr](http://download.openstreetmap.fr/replication/). Here are some examples:
-
-    `https://planet.openstreetmap.org/replication/minute/`
-    `http://download.openstreetmap.fr/replication/planet/minute/`
-    `http://download.openstreetmap.fr/replication/europe/minute/`
-    `http://download.openstreetmap.fr/replication/europe/netherlands/minute/`
-
-  - The update process is based on the sequence number comparison between the current **state.txt** file from the replication server, and the locally stored **lastState.txt**. So if you downloaded a osm.bz2 file, you should modify the **sequenceNumber** in the **lastState.txt** file accordingly.
-
-  - Execute the file **update_camera.sh** to import all surveillance entries that have been added between the creation of the osm.bz2 file and today.
-
-5. Schedule automatic database updates
-
-  - Add this line to your crontab:
-
-    `* * * * * /home/[user]/sunders/update_cameras/update_camera.sh > /dev/null 2>&1`
-
-  - Go to the directory **~/sunders/update_cameras/logs** to check if your schedule works. After one minute there should be a new log file.
-
-  - Go back to your crontab to change the schedule to the values you prefer, e.g. to `23 * * * *` to run the update at every 23rd minute past every hour.
-
-6. Configure the website
-
-  - Change to the directory **/var/www/sunders/**.
-
-  - Rename the file **config.php.example** to **config.php**.
-
-  - Open the file **config.php** and change the definitions of **DEFAULT_ZOOM**, **DEFAULT_LAT**, and **DEFAULT_LON** to set the initial focus of the map to the location you want. Furthermore you can set the **DEFAULT_LANGUAGE**.
-
-7. Check the website
-
-  - Enter the URL of your Surveillance under Surveillance instance (e.g. https://myserver/sunders/) into your browser.
-
-  - You should see a map with camera icons now.
-
-8. Optional: Add statistics table to your database
-
-  - Go to the directory **~/sunders/init_cameras/db/**.
-
-  - Execute the file **addStatistics.sql** for database user **camera**.
-
-    `mysql camera -h localhost -u camera --password=[camera user password] < addStatistics.sql`
-
-  - Open the file **~/sunders/update_cameras/config.php**. Change the value of **USE_STATISTICS** from **false** to **true** and enter the username of your GeoNames account at **WEBSERVICE_USER**.
-
-    `define('USE_STATISTICS', true);`
-
-  - Open the file **/var/www/sunders/config.php** and change the value of **USE_STATISTICS** from **false** to **true**.
-
-    `define('USE_STATISTICS', true);`
-
-  - Schedule automatic statistics updates by adding this new line to your crontab. Enter your preferred schedule values, e.g. `42 * * * *` to run the update at every 42nd minute past every hour.
-
-    `42 * * * * /home/[user]/sunders/update_cameras/update_statistics.sh > /dev/null 2>&1`
-
-  - Check the statistics by entering the URL of the visualization site (e.g. https://myserver/sunders/stats/) into your browser.
+_tbd.: Populate camera data_
 
 ## Surveillance nodes
 
@@ -198,38 +109,38 @@ Key | Value
 * The font [Grabstein Grotesk](https://fontlibrary.org/de/font/grabstein-grotesk) [OLF] is used for the titles.
 
 
-[img_red]: ./www/sunders/images/colorRed.png "Public surveillance"
-[img_redall]: ./www/sunders/images/colorRedAll.png "Public surveillance"
+[img_red]: ./web/www/sunders/images/colorRed.png "Public surveillance"
+[img_redall]: ./web/www/sunders/images/colorRedAll.png "Public surveillance"
 
-[img_blue]: ./www/sunders/images/colorBlue.png "Outdoor surveillance"
-[img_blueall]: ./www/sunders/images/colorBlueAll.png "Outdoor surveillance"
+[img_blue]: ./web/www/sunders/images/colorBlue.png "Outdoor surveillance"
+[img_blueall]: ./web/www/sunders/images/colorBlueAll.png "Outdoor surveillance"
 
-[img_green]: ./www/sunders/images/colorGreen.png "Indoor surveillance"
-[img_greenall]: ./www/sunders/images/colorGreenAll.png "Indoor surveillance"
+[img_green]: ./web/www/sunders/images/colorGreen.png "Indoor surveillance"
+[img_greenall]: ./web/www/sunders/images/colorGreenAll.png "Indoor surveillance"
 
-[img_cam]: ./www/sunders/images/cam.png "Camera"
-[img_camall]: ./www/sunders/images/camAll.png "Cameras"
-[img_todo_camall]: ./www/sunders/images/todo_camAll.png "Cameras, fixme"
+[img_cam]: ./web/www/sunders/images/cam.png "Camera"
+[img_camall]: ./web/www/sunders/images/camAll.png "Cameras"
+[img_todo_camall]: ./web/www/sunders/images/todo_camAll.png "Cameras, fixme"
 
-[img_fixed]: ./www/sunders/images/fixed.png "Fixed camera"
-[img_fixedall]: ./www/sunders/images/fixedAll.png "Fixed cameras"
-[img_todo_fixedall]: ./www/sunders/images/todo_fixedAll.png "Fixed cameras, fixme"
+[img_fixed]: ./web/www/sunders/images/fixed.png "Fixed camera"
+[img_fixedall]: ./web/www/sunders/images/fixedAll.png "Fixed cameras"
+[img_todo_fixedall]: ./web/www/sunders/images/todo_fixedAll.png "Fixed cameras, fixme"
 
-[img_panning]: ./www/sunders/images/panning.png "Panning camera"
-[img_panningall]: ./www/sunders/images/panningAll.png "Panning cameras"
-[img_todo_panningall]: ./www/sunders/images/todo_panningAll.png "Panning cameras, fixme"
+[img_panning]: ./web/www/sunders/images/panning.png "Panning camera"
+[img_panningall]: ./web/www/sunders/images/panningAll.png "Panning cameras"
+[img_todo_panningall]: ./web/www/sunders/images/todo_panningAll.png "Panning cameras, fixme"
 
-[img_dome]: ./www/sunders/images/dome.png "Dome camera"
-[img_domeall]: ./www/sunders/images/domeAll.png "Dome cameras"
-[img_todo_domeall]: ./www/sunders/images/todo_domeAll.png "Dome cameras, fixme"
+[img_dome]: ./web/www/sunders/images/dome.png "Dome camera"
+[img_domeall]: ./web/www/sunders/images/domeAll.png "Dome cameras"
+[img_todo_domeall]: ./web/www/sunders/images/todo_domeAll.png "Dome cameras, fixme"
 
-[img_guard]: ./www/sunders/images/guard.png "Guard"
-[img_guardall]: ./www/sunders/images/guardAll.png "Guards"
-[img_todo_guardall]: ./www/sunders/images/todo_guardAll.png "Guards, fixme"
+[img_guard]: ./web/www/sunders/images/guard.png "Guard"
+[img_guardall]: ./web/www/sunders/images/guardAll.png "Guards"
+[img_todo_guardall]: ./web/www/sunders/images/todo_guardAll.png "Guards, fixme"
 
-[img_traffic]: ./www/sunders/images/traffic.png "Automatic Licence Plate Recognition"
-[img_todo_traffic]: ./www/sunders/images/todo_traffic.png "Automatic Licence Plate Recognition, fixme"
+[img_traffic]: ./web/www/sunders/images/traffic.png "Automatic Licence Plate Recognition"
+[img_todo_traffic]: ./web/www/sunders/images/todo_traffic.png "Automatic Licence Plate Recognition, fixme"
 
-[img_example1]: ./www/sunders/images/fixed_z-17_d-90_a-15_h-5.png "Field of view (direction: 90°, angle: 15°, height: 5m)"
-[img_example2]: ./www/sunders/images/fixed_z-17_d-90_a-15_h-10.png "Field of view (direction: 90°, angle: 15°, height: 10m)"
-[img_example3]: ./www/sunders/images/fixed_z-17_d-90_a-60_h-10.png "Field of view (direction: 90°, angle: 60°, height: 10m)"
+[img_example1]: ./web/www/sunders/images/fixed_z-17_d-90_a-15_h-5.png "Field of view (direction: 90°, angle: 15°, height: 5m)"
+[img_example2]: ./web/www/sunders/images/fixed_z-17_d-90_a-15_h-10.png "Field of view (direction: 90°, angle: 15°, height: 10m)"
+[img_example3]: ./web/www/sunders/images/fixed_z-17_d-90_a-60_h-10.png "Field of view (direction: 90°, angle: 60°, height: 10m)"

data_handler/Containerfile

@@ -0,0 +1,8 @@
+FROM quay.io/fedora/fedora-minimal:43
+RUN microdnf install -y php-cli php-mysqli php-bcmath php-xml python-lxml gunzip https://raw.githubusercontent.com/rpmsphere/x86_64/master/d/dcron-4.5-7.1.x86_64.rpm && microdnf clean all
+COPY ./utils/* /opt/
+COPY ./data_init/*.sql /opt/init/init.sql
+
+ENTRYPOINT ["/opt/entrypoint.sh"]
+
+CMD ["/usr/sbin/dcrond", "-f" ]

data_handler/utils/config.php

@@ -2,10 +2,10 @@
 
 define('REPLICATE_URL', 'https://planet.openstreetmap.org/replication/minute');
 
-define('MYSQL_HOST', 'localhost');
-define('MYSQL_DB', 'camera');
-define('MYSQL_USER', 'camera');
-define('MYSQL_PASSWD', 'xxxxxxxx');
+define('MYSQL_HOST', getenv('MYSQL_HOST'));
+define('MYSQL_DB', getenv('MYSQL_DB'));
+define('MYSQL_USER', getenv('MYSQL_USER'));
+define('MYSQL_PASSWORD', getenv('MYSQL_PASSWORD'));
 
 define('USE_STATISTICS', false);
 define('WEBSERVICE_COUNTRY_URL', 'http://api.geonames.org/countryCode');

data_handler/utils/create_camera_update_statements.php

@@ -0,0 +1,220 @@
+<?php
+
+include "config.php";
+
+$elementTypes = array();
+$count = 0;
+$countDelete = 0;
+$countModifyDelete = 0;
+$countModify = 0;
+$countCreate = 0;
+$mode = '';
+$curNodeAttrs = null;
+$curNodeTags = null;
+$id = 0;
+$latitude = 0;
+$longitude = 0;
+
+$sql_statement = "BEGIN TRANSACTION;\n";
+
+$mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB);
+
+if($mysqli->connect_errno) {
+  echo "Error while connecting to DB : $mysqli->error \n" ;
+  exit(1);
+}
+
+
+if (! ($deleteStmt = $mysqli->prepare("DELETE FROM position WHERE id=?"))) {
+  echo "Error while preparing delete position statement : " . $mysqli->error ;
+  exit(1);
+}
+$deleteStmt->bind_param('d', $id);
+
+if (! ($deleteTagStmt = $mysqli->prepare("DELETE FROM tag WHERE id=?"))) {
+  echo "Error while preparing delete tag statement : " . $mysqli->error ;
+  exit(1);
+}
+$deleteTagStmt->bind_param('d', $id);
+
+if (USE_STATISTICS) {
+  if (! ($deleteStatsStmt = $mysqli->prepare("DELETE FROM statistics WHERE id=?"))) {
+    echo "Error while preparing delete statistics statement : " . $mysqli->error ;
+    exit(1);
+  }
+  $deleteStatsStmt->bind_param('d', $id);
+}
+
+if (! ($insertStmt = $mysqli->prepare("INSERT INTO position (id, latitude, longitude) VALUES (?, ?, ?)"))) {
+  echo "Error while preparing insert position statement : " . $mysqli->error ;
+  exit(1);
+}
+$insertStmt->bind_param('dii', $id, $latitude, $longitude);
+
+if (! ($insertTagStmt = $mysqli->prepare("INSERT INTO tag (id, k, v) VALUES (?, ?, ?)"))) {
+  echo "Error while preparing insert tag statement : " . $mysqli->error ;
+  exit(1);
+}
+$insertTagStmt->bind_param('dss', $id, $k, $v);
+
+
+function startElement ($parser, $name, $attrs) {
+  global $elementTypes, $count, $mode, $curNodeAttrs, $curNodeTags;
+  $count++;
+
+  if (array_key_exists($name, $elementTypes)) {
+    $elementTypes[$name] += 1;
+  } else {
+    $elementTypes[$name] = 1;
+  }
+
+  if ($name == 'modify' || $name == 'delete' or $name == 'create') {
+    $mode = $name;
+
+  } else if ($name == 'node') {
+    $curNodeAttrs = $attrs;
+    $curNodeTags = array();
+
+  } else if ($name == 'tag') {
+    $curNodeTags[$attrs['k']] = $attrs['v'];
+  }
+}
+
+function endElement ($parser, $name) {
+  if (USE_STATISTICS) {
+    global $mode, $deleteStmt, $deleteTagStmt, $deleteStatsStmt, $insertStmt, $insertTagStmt, $insertStatsStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $ts, $version, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate, $sql_statement;
+  } else {
+    global $mode, $deleteStmt, $deleteTagStmt, $insertStmt, $insertTagStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate, $sql_statement;
+  }
+
+  if ($name == 'modify' || $name == 'delete' or $name == 'create') {
+    $mode = '';
+  } else if ($name == 'node') {
+    if ($mode == 'delete') {
+      $id = $curNodeAttrs['id'];
+
+      if (! $deleteTagStmt->execute()) {
+        echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n";
+      }
+      
+      if (! $deleteStmt->execute()) {
+        echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n";
+      }
+      if ($deleteStmt->affected_rows > 0) {
+         if (! $deleteTagStmt->execute()) {
+        echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n";
+      }
+      
+      if (! $deleteStmt->execute()) {
+        echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n";
+      }
+        $sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
+        $sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
+      }
+
+    } else if ($mode == 'modify' || $mode == 'create') {
+      if (! empty($curNodeTags)
+          && array_key_exists('man_made', $curNodeTags)
+          && $curNodeTags['man_made'] == 'surveillance') {
+
+        
+
+        $id = $curNodeAttrs['id'];
+
+        if ($mode == 'modify') {
+          if (! $deleteTagStmt->execute()) {
+            echo "***** Error : Deleting tags $id for modification : ". $deleteTagStmt->error . "\n";
+          }
+          if (! $deleteStmt->execute()) {
+            echo "***** Error : Deleting $id for modification : ". $deleteStmt->error . "\n";
+          }
+          $sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
+          $sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
+        } else {
+        }
+
+        $latitude = (int) ($curNodeAttrs['lat'] * 10000000);
+        $longitude = (int) ($curNodeAttrs['lon'] * 10000000);
+        $sql_statement = $sql_statement . "INSERT INTO position (id, latitude, longitude) VALUES ($id, $latitude , $longitude);\n";
+        
+
+        $k = 'lat';
+        $v = $curNodeAttrs['lat'];
+        $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
+        
+
+        $k = 'lon';
+        $v = $curNodeAttrs['lon'];
+        $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
+
+
+        $k = 'userid';
+        $v = $curNodeAttrs['user'];
+        $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
+
+
+        $k = 'version';
+        $v = $curNodeAttrs['version'];
+        $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
+       
+
+
+        $k = 'timestamp';
+        $v = $curNodeAttrs['timestamp'];
+        $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
+
+        foreach($curNodeTags as $k => $v) {
+        $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
+          
+        }
+      } else if ($mode == 'modify') {
+        // delete former surveillance nodes that were modified to non-surveillance nodes
+        $id = $curNodeAttrs['id'];
+
+        if (! $deleteTagStmt->execute()) {
+          echo "***** Error : Deleting tags $id for non-surveillance node : ". $deleteTagStmt->error . "\n";
+        }
+        if (! $deleteStmt->execute()) {
+          echo "***** Error : Deleting $id for non-surveillance node : ". $deleteStmt->error . "\n";
+        }
+
+        if ($deleteStmt->affected_rows > 0) {
+          $sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
+          $sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
+        }
+      }
+    }
+
+    $curNodeAttrs = null;
+    $curNodeTags = null;
+  }
+}
+
+$file="change_file.osc";
+
+$xml_parser = xml_parser_create();
+xml_parser_set_option($xml_parser, XML_OPTION_CASE_FOLDING, false);
+xml_set_element_handler($xml_parser, "startElement", "endElement");
+
+if (!($fp = fopen($file, "r"))) {
+  die("could not open XML input");
+}
+
+while ($data = fread($fp, 4096)) {
+  if (!xml_parse($xml_parser, $data, feof($fp))) {
+    die(sprintf("XML error: %s at line %d",
+                xml_error_string(xml_get_error_code($xml_parser)),
+                xml_get_current_line_number($xml_parser)));
+  }
+}
+
+$sql="UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';";
+$mysqli->query($sql);
+$sql_statement = $sql_statement . "UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';\n";
+$sql_statement = $sql_statement . "COMMIT;";
+$sql_file = "/migrations/".$_SERVER['argv'][1].".sql";
+
+file_put_contents($sql_file, $sql_statement);
+$mysqli->close();
+
+?>

data_handler/utils/create_camera_update_statements_download.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+exeDir=$(cd "$(dirname "$0")"; pwd)
+cd "$exeDir"
+
+REPLICATE_URL=$(grep -oP "REPLICATE_URL[',\s]*['\"]\Khttps?://[^'\"]+" config.php)
+
+echo "Using replicate URL: $REPLICATE_URL"
+export REPLICATE_URL
+
+
+lastSeqNum=$(php get_sync_state.php)
+curl -sL "$REPLICATE_URL/state.txt" -o state.txt
+newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-)
+echo $newSeqNum
+if [ "$newSeqNum" -eq "$lastSeqNum" ]; then
+  echo "No new file to be processed"
+  exit 0
+fi
+
+mkdir -p logs
+seq "$((lastSeqNum + 1))" "$newSeqNum" > seq_list.txt
+
+
+cat seq_list.txt | xargs -n1 -P10 -I{} bash -c '
+  seqnum="$1"
+  f=$(printf "%09d" "$seqnum")
+  path=$(echo "$f" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
+  url="${REPLICATE_URL}/${path}.osc.gz"
+  echo "Downloading $url"
+  curl -sL "$url" -o /migrations/"${seqnum}.osc.gz" && gunzip /migrations/"${seqnum}.osc.gz" && python filter_osc.py ${seqnum} && rm /migrations/${seqnum}.osc 
+' _ {}

data_handler/utils/create_camera_update_statements_process.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+exedir=$(cd `dirname "$0"`; pwd)
+
+REPLICATE_URL=`grep "REPLICATE_URL" "$exedir/config.php"  | sed -e 's/.*http/http/' -e 's#...$##' `
+
+if [ -e "/var/lock/update_camera" ]
+then
+  # Maybe an other update is running
+  otherPid=$(cat "/var/lock/update_camera")
+  count=$(ps $otherPid | grep -c `basename "$0"`)
+
+  if [ $count -gt 0 ]
+  then
+    echo "$0 is running yet. Exiting." >&2
+    exit 1
+  fi
+fi
+
+# OK. We can update the database...
+echo $$ >  "/var/lock/update_camera"
+
+exeDir=$(cd `dirname "$0"`; pwd);
+cd "$exeDir"
+
+if [ -e "state.txt" ]
+then
+  rm "state.txt"
+fi
+
+#TODO get state from db
+# Read the last update timestamp
+# lastTimestamp=$(grep "^timestamp=" "lastState.txt" | cut -d'=' -f2-)
+# lastSeqNum=$(grep "^sequenceNumber=" "lastState.txt" | cut -d'=' -f2-)
+
+lastSeqNum=$(php get_sync_state.php)
+
+curl -L "$REPLICATE_URL/state.txt" -o state.txt 
+
+newTimestamp=$(grep "^timestamp=" "state.txt" | cut -d'=' -f2-)
+newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-)
+
+if [ $newSeqNum -eq $lastSeqNum ]
+then
+  echo "No new file to be processed"
+  exit 1
+fi
+mkdir logs
+curSeqNum=$lastSeqNum
+curSeqNum=$(( $curSeqNum + 1 ))
+while [ $curSeqNum -lt $newSeqNum ]
+do
+  if [ -e /migrations/"${curSeqNum}_filtered.osc" ]
+  then
+  
+
+  rm "change_file.osc"
+  cp /migrations/${curSeqNum}_filtered.osc change_file.osc
+
+  php create_camera_update_statements.php $curSeqNum
+  rm /migrations/${curSeqNum}_filtered.osc
+  curSeqNum=$(( $curSeqNum + 1 ))
+  else
+  path=$(echo "$(printf "%09d" "$curSeqNum")" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
+  url="${REPLICATE_URL}/${path}.osc.gz"
+  echo $url
+  curl -sL "$url" -o /migrations/"$curSeqNum.osc.gz" && gunzip /migrations/"$curSeqNum.osc.gz" && python filter_osc.py ${curSeqNum} && rm /migrations/${curSeqNum}.osc 
+  fi
+done
+
+rm "/var/lock/update_camera"

data_handler/utils/entrypoint.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+set -e
+
+
+####################################################################################################
+### Crontab settings
+####################################################################################################
+
+cat<<EOF | dcrontab -
+# min     hour      day     month     weekday command
+5    *   *       *         *       sh /opt/update_camera.sh
+EOF
+
+####################################################################################################
+### Run prescripts
+####################################################################################################
+
+php /opt/init_db.php
+#TODO add db_migrations
+echo "Prescripts done"
+
+####################################################################################################
+### Run cmd
+####################################################################################################
+
+# see: https://github.com/dubiousjim/dcron/issues/13
+# ignore using `exec` for `dcron` to get another pid instead of `1`
+# exec "$@"
+"$@"

data_handler/utils/filter_osc.py

@@ -0,0 +1,90 @@
+import sys
+import gzip
+from lxml import etree
+
+def open_maybe_gzip(filename, mode="rb"):
+    """Open file normally or as gzip based on extension."""
+    if filename.endswith(".gz"):
+        return gzip.open(filename, mode)
+    return open(filename, mode)
+
+def process_file(number):
+    input_file = f"/migrations/{number}.osc"
+    filtered_file = f"/migrations/{number}_filtered.osc"
+    deleted_file = f"/migrations/{number}_deleted.osc"
+
+    # --- open input and outputs ---
+    with open_maybe_gzip(input_file, "rb") as infile, \
+         open(filtered_file, "wb") as out_filt, \
+         open(deleted_file, "wb") as out_del:
+
+        # XML headers
+        out_filt.write(b"<?xml version='1.0' encoding='UTF-8'?>\n")
+        out_filt.write(b"<osmChange version='0.6' generator='filter-script'>\n")
+
+        out_del.write(b"<?xml version='1.0' encoding='UTF-8'?>\n")
+        out_del.write(b"<osmChange version='0.6' generator='filter-script'>\n")
+
+        context = etree.iterparse(infile, events=("start", "end"))
+        current_section = None
+        buffer_filt = None
+        buffer_del = None
+
+        for event, elem in context:
+            if event == "start" and elem.tag in ("create", "modify", "delete"):
+                current_section = elem.tag
+                if current_section == "delete":
+                    buffer_del = etree.Element("delete")
+                else:
+                    buffer_filt = etree.Element(current_section)
+
+            elif event == "end" and elem.tag == "node" and current_section:
+                node_copy = etree.fromstring(etree.tostring(elem))
+
+                if current_section in ("create", "modify"):
+                    tags = {t.get("k"): t.get("v") for t in elem.findall("tag")}
+                    if tags.get("surveillance:type") == "camera":
+                        buffer_filt.append(node_copy)
+
+                elif current_section == "delete":
+                    buffer_del.append(node_copy)
+
+                elem.clear()
+
+            elif event == "end" and elem.tag in ("create", "modify"):
+                if buffer_filt is not None and len(buffer_filt):
+                    out_filt.write(f"  <{current_section}>\n".encode())
+                    for node in buffer_filt:
+                        out_filt.write(etree.tostring(node, encoding="utf-8"))
+                        out_filt.write(b"\n")
+                    out_filt.write(f"  </{current_section}>\n".encode())
+                buffer_filt = None
+                current_section = None
+                elem.clear()
+
+            elif event == "end" and elem.tag == "delete":
+                if buffer_del is not None and len(buffer_del):
+                    out_del.write(b"  <delete>\n")
+                    for node in buffer_del:
+                        out_del.write(etree.tostring(node, encoding="utf-8"))
+                        out_del.write(b"\n")
+                    out_del.write(b"  </delete>\n")
+                buffer_del = None
+                current_section = None
+                elem.clear()
+
+        out_filt.write(b"</osmChange>\n")
+        out_del.write(b"</osmChange>\n")
+
+    print(f"✅ Processed {input_file}")
+    print(f"   → {filtered_file} (camera nodes)")
+    print(f"   → {deleted_file} (delete nodes)")
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python filter_replication.py <replication_number>")
+        print("Example: python filter_replication.py 123")
+        sys.exit(1)
+
+    number = sys.argv[1]
+    process_file(number)

data_handler/utils/get_sync_state.php

@@ -0,0 +1,21 @@
+<?php
+
+include "config.php";
+
+$mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB);
+
+if($mysqli->connect_errno) {
+  echo "Error while connecting to DB : $mysqli->error \n" ;
+  exit(1);
+}
+$mysqli->autocommit(FALSE);
+
+$result = $mysqli->query("SELECT * FROM sync_state WHERE k = 'sequenceNumber'");
+
+while($row = $result->fetch_assoc()) {
+    echo $row["v"];
+  }
+
+$mysqli->close();
+
+?>

data_handler/utils/init_db.php

@@ -0,0 +1,98 @@
+<?php
+ini_set("memory_limit", "400M");
+// Admin credentials (must have privileges to create DB/users)
+$dbHost     = getenv('MYSQL_HOST');
+$dbAdmin    = getenv('MYSQL_USER');
+$dbPassword = getenv('MYSQL_PASSWORD');
+$dbName = getenv('MYSQL_DB');
+
+// Variables for new users
+$cameraUser        = getenv('CAMERA_USER');
+$cameraPassword    = getenv('CAMERA_USER_PASSWORD');
+
+$camSelectUser     = getenv('CAMERA_SELECT_USER');
+$camSelectPassword = getenv('CAMERA_SELECT_USER_PASSWORD');
+
+// Connect to MySQL
+$conn = new mysqli($dbHost, $dbAdmin, $dbPassword);
+
+// Check connection
+if ($conn->connect_error) {
+    die("Connection failed: " . $conn->connect_error);
+}
+
+
+if (!$conn->query("CREATE DATABASE IF NOT EXISTS $dbName")) {
+    echo "Error creating database: " . $conn->error . "\n";
+}
+if (!$conn->select_db($dbName)) {
+    die("Error selecting database: " . $conn->error);
+}
+
+
+// Create camera user
+$conn->query("CREATE USER IF NOT EXISTS '$cameraUser'@'%' IDENTIFIED BY '$cameraPassword'");
+$conn->query("GRANT ALL PRIVILEGES ON $dbName.* TO '$cameraUser'@'%'");
+
+// Create camselect user
+$conn->query("CREATE USER IF NOT EXISTS '$camSelectUser'@'%' IDENTIFIED BY '$camSelectPassword'");
+$conn->query("GRANT SELECT ON $dbName.* TO '$camSelectUser'@'%'");
+
+
+// position table
+$conn->query("
+CREATE TABLE IF NOT EXISTS position (
+  id BIGINT PRIMARY KEY,
+  latitude INT,
+  longitude INT
+)
+");
+
+// tag table
+$conn->query("
+CREATE TABLE IF NOT EXISTS tag (
+  id BIGINT,
+  k VARCHAR(100),
+  v VARCHAR(10000),
+  PRIMARY KEY (id, k),
+  CONSTRAINT fk_position FOREIGN KEY (id) REFERENCES `position`(id) ON DELETE CASCADE
+)
+");
+
+$conn->query("CREATE INDEX IF NOT EXISTS LatLon ON position (latitude, longitude)");
+
+
+// sync_state table
+$conn->query("
+CREATE TABLE IF NOT EXISTS sync_state (
+  k VARCHAR(100),
+  v VARCHAR(100)
+  )
+");
+
+$result = $conn -> query("SELECT * FROM sync_state WHERE k='sequenceNumber'");
+
+$table_count = $result -> num_rows;
+
+if ($table_count == 0){
+$conn->query("INSERT INTO sync_state (k, v) VALUES ('sequenceNumber', '0');");
+}
+
+echo "Database, users, and tables created successfully.\n";
+
+$result = $conn -> query("SELECT * FROM position limit 10");
+
+$table_count = $result -> num_rows;
+
+printf("Result set has %d rows.\n", $table_count);
+if ($table_count == 0){
+$location = "/opt/init/init.sql";
+
+$commands = file_get_contents($location);   
+$conn->multi_query($commands);
+echo "Inserted data.\n";
+}
+
+
+$conn->close();
+?>

data_handler/utils/update_camera.php

@@ -0,0 +1,291 @@
+<?php
+
+include "config.php";
+
+$elementTypes = array();
+$count = 0;
+$countDelete = 0;
+$countModifyDelete = 0;
+$countModify = 0;
+$countCreate = 0;
+$mode = '';
+$curNodeAttrs = null;
+$curNodeTags = null;
+$id = 0;
+$latitude = 0;
+$longitude = 0;
+
+$mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB);
+
+if($mysqli->connect_errno) {
+  echo "Error while connecting to DB : $mysqli->error \n" ;
+  exit(1);
+}
+$mysqli->autocommit(FALSE);
+
+if (! ($deleteStmt = $mysqli->prepare("DELETE FROM position WHERE id=?"))) {
+  echo "Error while preparing delete position statement : " . $mysqli->error ;
+  exit(1);
+}
+$deleteStmt->bind_param('d', $id);
+
+if (! ($deleteTagStmt = $mysqli->prepare("DELETE FROM tag WHERE id=?"))) {
+  echo "Error while preparing delete tag statement : " . $mysqli->error ;
+  exit(1);
+}
+$deleteTagStmt->bind_param('d', $id);
+
+if (USE_STATISTICS) {
+  if (! ($deleteStatsStmt = $mysqli->prepare("DELETE FROM statistics WHERE id=?"))) {
+    echo "Error while preparing delete statistics statement : " . $mysqli->error ;
+    exit(1);
+  }
+  $deleteStatsStmt->bind_param('d', $id);
+}
+
+if (! ($insertStmt = $mysqli->prepare("INSERT INTO position (id, latitude, longitude) VALUES (?, ?, ?)"))) {
+  echo "Error while preparing insert position statement : " . $mysqli->error ;
+  exit(1);
+}
+$insertStmt->bind_param('dii', $id, $latitude, $longitude);
+
+if (! ($insertTagStmt = $mysqli->prepare("INSERT INTO tag (id, k, v) VALUES (?, ?, ?)"))) {
+  echo "Error while preparing insert tag statement : " . $mysqli->error ;
+  exit(1);
+}
+$insertTagStmt->bind_param('dss', $id, $k, $v);
+
+if (USE_STATISTICS) {
+  if (! ($insertStatsStmt = $mysqli->prepare("INSERT INTO statistics (id, ts, version) VALUES (?, ?, ?)"))) {
+    echo "Error while preparing insert statistics statement : " . $mysqli->error ;
+    exit(1);
+  }
+  $insertStatsStmt->bind_param('dsi', $id, $ts, $version);
+}
+
+function printDebug() {
+  global $elementTypes, $count, $countDelete, $countModifyDelete, $countModify, $countCreate;
+
+  echo "== $count ==============================\n";
+  foreach($elementTypes as $k => $v) {
+    echo "$k : $v\n";
+  }
+  echo "++++++ Surveillance nodes : ++++++\n";
+  echo "$countDelete deletions\n";
+  echo "$countModifyDelete modifications --> deletions\n";
+  echo "$countModify modifications\n";
+  echo "$countCreate creations\n";
+}
+
+function printDebugCurNode() {
+  global $curNodeAttrs, $curNodeTags, $mode;
+
+  echo "=============\n";
+  echo "$mode : " . $curNodeAttrs['id'] ." (". $curNodeAttrs['lat'] . " x " . $curNodeAttrs['lon'] . ") :  " . $curNodeAttrs['user'] . "\n";
+  if (! empty($curNodeAttrs)) {
+    echo "  => Attributes :\n";
+    foreach($curNodeAttrs as $k => $v) {
+      echo "   $k : $v\n";
+    }
+  } else {
+     echo "  => No attributes\n";
+  }
+  if (! empty($curNodeTags)) {
+    echo "  => Tags :\n";
+    foreach($curNodeTags as $k => $v) {
+      echo "   $k : $v\n";
+    }
+  } else {
+     echo "  => No tags\n";
+  }
+}
+
+function startElement ($parser, $name, $attrs) {
+  global $elementTypes, $count, $mode, $curNodeAttrs, $curNodeTags;
+  $count++;
+
+  if (array_key_exists($name, $elementTypes)) {
+    $elementTypes[$name] += 1;
+  } else {
+    $elementTypes[$name] = 1;
+  }
+
+  if ($name == 'modify' || $name == 'delete' or $name == 'create') {
+    $mode = $name;
+
+  } else if ($name == 'node') {
+    $curNodeAttrs = $attrs;
+    $curNodeTags = array();
+
+  } else if ($name == 'tag') {
+    $curNodeTags[$attrs['k']] = $attrs['v'];
+  }
+}
+
+function endElement ($parser, $name) {
+  if (USE_STATISTICS) {
+    global $mode, $deleteStmt, $deleteTagStmt, $deleteStatsStmt, $insertStmt, $insertTagStmt, $insertStatsStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $ts, $version, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate;
+  } else {
+    global $mode, $deleteStmt, $deleteTagStmt, $insertStmt, $insertTagStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate;
+  }
+
+  if ($name == 'modify' || $name == 'delete' or $name == 'create') {
+    $mode = '';
+  } else if ($name == 'node') {
+    if ($mode == 'delete') {
+      $id = $curNodeAttrs['id'];
+
+      if (USE_STATISTICS) {
+        if (! $deleteStatsStmt->execute()) {
+          echo "***** Error : Deleting statistics $id : ". $deleteStatsStmt->error . "\n";
+        }
+      }
+
+      if (! $deleteTagStmt->execute()) {
+        echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n";
+      }
+      if (! $deleteStmt->execute()) {
+        echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n";
+      }
+      if ($deleteStmt->affected_rows > 0) {
+        $countDelete++;
+        $mysqli->commit();
+        printDebugCurNode();
+      }
+
+    } else if ($mode == 'modify' || $mode == 'create') {
+      if (! empty($curNodeTags)
+          && array_key_exists('man_made', $curNodeTags)
+          && $curNodeTags['man_made'] == 'surveillance') {
+
+        printDebugCurNode();
+
+        $id = $curNodeAttrs['id'];
+
+        if ($mode == 'modify') {
+          $countModify++;
+
+          if (USE_STATISTICS) {
+            if (! $deleteStatsStmt->execute()) {
+              echo "***** Error : Deleting statistics $id for modification : ". $deleteStatsStmt->error . "\n";
+            }
+          }
+
+          if (! $deleteTagStmt->execute()) {
+            echo "***** Error : Deleting tags $id for modification : ". $deleteTagStmt->error . "\n";
+          }
+          if (! $deleteStmt->execute()) {
+            echo "***** Error : Deleting $id for modification : ". $deleteStmt->error . "\n";
+          }
+        } else {
+          $countCreate++;
+        }
+
+        $latitude = (int) ($curNodeAttrs['lat'] * 10000000);
+        $longitude = (int) ($curNodeAttrs['lon'] * 10000000);
+
+        if (! $insertStmt->execute()) {
+          echo "***** Error : inserting $id ($latitude x $longitude) : ". $insertStmt->error . "\n";
+        }
+
+        $k = 'lat';
+        $v = $curNodeAttrs['lat'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting latitude $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        $k = 'lon';
+        $v = $curNodeAttrs['lon'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting longitude $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        $k = 'userid';
+        $v = $curNodeAttrs['user'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting user $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        $k = 'version';
+        $v = $curNodeAttrs['version'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting version $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        $k = 'timestamp';
+        $v = $curNodeAttrs['timestamp'];
+        if (! $insertTagStmt->execute()) {
+          echo "***** Error : inserting timestamp $v for $id : ". $insertTagStmt->error . "\n";
+        }
+
+        foreach($curNodeTags as $k => $v) {
+          if (! $insertTagStmt->execute()) {
+            echo "***** Error : inserting tag $k => $v for $id : ". $insertTagStmt->error . "\n";
+          }
+        }
+
+        if (USE_STATISTICS) {
+          $ts = $curNodeAttrs['timestamp'];
+          $version = $curNodeAttrs['version'];
+          if (! $insertStatsStmt->execute()) {
+            echo "***** Error : inserting ts $ts, version $version for $id : ". $insertStatsStmt->error . "\n";
+          }
+        }
+
+        $mysqli->commit();
+      } else if ($mode == 'modify') {
+        // delete former surveillance nodes that were modified to non-surveillance nodes
+        $id = $curNodeAttrs['id'];
+
+        if (USE_STATISTICS) {
+          if (! $deleteStatsStmt->execute()) {
+            echo "***** Error : Deleting statistics $id for non-surveillance node : ". $deleteStatsStmt->error . "\n";
+          }
+        }
+        if (! $deleteTagStmt->execute()) {
+          echo "***** Error : Deleting tags $id for non-surveillance node : ". $deleteTagStmt->error . "\n";
+        }
+        if (! $deleteStmt->execute()) {
+          echo "***** Error : Deleting $id for non-surveillance node : ". $deleteStmt->error . "\n";
+        }
+
+        if ($deleteStmt->affected_rows > 0) {
+          $countModifyDelete++;
+          $mysqli->commit();
+          printDebugCurNode();
+          echo "  ==> delete because modified to non-surveillance node\n";
+        }
+      }
+    }
+
+    $curNodeAttrs = null;
+    $curNodeTags = null;
+  }
+}
+
+$file="change_file.osc";
+
+$xml_parser = xml_parser_create();
+xml_parser_set_option($xml_parser, XML_OPTION_CASE_FOLDING, false);
+xml_set_element_handler($xml_parser, "startElement", "endElement");
+
+if (!($fp = fopen($file, "r"))) {
+  die("could not open XML input");
+}
+
+while ($data = fread($fp, 4096)) {
+  if (!xml_parse($xml_parser, $data, feof($fp))) {
+    die(sprintf("XML error: %s at line %d",
+                xml_error_string(xml_get_error_code($xml_parser)),
+                xml_get_current_line_number($xml_parser)));
+  }
+}
+
+$sql="UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';";
+$mysqli->query($sql);
+$mysqli->commit();
+
+printDebug();
+$mysqli->close();
+
+?>

data_handler/utils/update_camera.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+exedir=$(cd `dirname "$0"`; pwd)
+
+REPLICATE_URL=`grep "REPLICATE_URL" "$exedir/config.php"  | sed -e 's/.*http/http/' -e 's#...$##' `
+
+if [ -e "/var/lock/update_camera" ]
+then
+  # Maybe an other update is running
+  otherPid=$(cat "/var/lock/update_camera")
+  count=$(ps $otherPid | grep -c `basename "$0"`)
+
+  if [ $count -gt 0 ]
+  then
+    echo "$0 is running yet. Exiting." >&2
+    exit 1
+  fi
+fi
+
+# OK. We can update the database...
+echo $$ >  "/var/lock/update_camera"
+
+exeDir=$(cd `dirname "$0"`; pwd);
+cd "$exeDir"
+
+if [ -e "state.txt" ]
+then
+  rm "state.txt"
+fi
+
+#TODO get state from db
+# Read the last update timestamp
+# lastTimestamp=$(grep "^timestamp=" "lastState.txt" | cut -d'=' -f2-)
+# lastSeqNum=$(grep "^sequenceNumber=" "lastState.txt" | cut -d'=' -f2-)
+
+lastSeqNum=$(php get_sync_state.php)
+
+curl -L "$REPLICATE_URL/state.txt" -o state.txt 
+
+newTimestamp=$(grep "^timestamp=" "state.txt" | cut -d'=' -f2-)
+newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-)
+
+if [ $newSeqNum -eq $lastSeqNum ]
+then
+  echo "No new file to be processed"
+  exit 1
+fi
+mkdir logs
+curSeqNum=$lastSeqNum
+while [ $curSeqNum -lt $newSeqNum ]
+do
+  curSeqNum=$(( $curSeqNum + 1 ))
+
+  logFileName="logs/log.$curSeqNum"
+
+  if [ -e "change_file.osc" ]
+  then
+    rm "change_file.osc"
+  fi
+
+  if [ -e "change_file.osc.gz" ]
+  then
+    rm "change_file.osc.gz"
+  fi
+
+  echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName"
+
+  targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'`
+  targetDirName="$REPLICATE_URL/$targetDirName"
+
+  curl -L "$targetDirName" -o change_file.osc.gz 2>&1 | tee -a $logFileName
+  if [ $? -gt 0 ]
+  then
+    echo "Error during recovery of $targetDirName" | tee -a $logFileName
+    exit 1
+  fi
+
+  gunzip change_file.osc.gz 2>&1 | tee -a $logFileName
+  if [ $? -gt 0 ]
+  then
+    echo "Error during decompression of $targetDirName" | tee -a $logFileName
+    exit 1
+  fi
+
+  php update_camera.php $curSeqNum 2>&1 | tee -a $logFileName
+
+  targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'`
+
+  curl -L "$targetDirName" -o lastState.txt 2>&1 | tee -a $logFileName
+  echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName"
+done
+
+rm "/var/lock/update_camera"

data_handler_python/Containerfile

@@ -0,0 +1,4 @@
+FROM quay.io/fedora/fedora-minimal:43
+RUN microdnf install -y python-pip && microdnf clean all && pip install OSMPythonTools
+
+COPY ./lookup_country.py /opt/

data_handler_python/lookup_country.py

@@ -0,0 +1,7 @@
+from OSMPythonTools.nominatim import Nominatim
+
+
+nominatim = Nominatim()
+heidelberg = nominatim.query(49.4093582, 8.694724, reverse=True, zoom=10)
+
+print(heidelberg.address()['country_code'])

data_handler_python/test.py

@@ -0,0 +1,12 @@
+
+
+import xml.etree.cElementTree as ET
+import gzip
+import requests
+
+r = requests.get('https://planet.openstreetmap.org/replication/minute/000/000/001.osc.gz')
+
+data = gzip.decompress(r.content)
+
+tree = ET.fromstring(data)
+print(tree.findall("./."))

docker-compose.yml

@@ -0,0 +1,57 @@
+services:
+  db:
+    image: mariadb:12.0.2
+    command: --max_allowed_packet=3250585600
+    environment:
+      MYSQL_ROOT_PASSWORD: rootpassword # ${{secrets.MYSQL_ROOT_PASSWORD}}
+      MYSQL_DATABASE: camera # ${{secrets.MYSQL_DATABASE}}
+      MYSQL_USER: camera # ${{secrets.MYSQL_USER}}
+      MYSQL_PASSWORD: camerapassword # ${{secrets.MYSQL_PASSWORD}}
+    ports:
+      - "3306:3306"
+    volumes:
+        - mariadb:/var/lib/mysql:Z
+    healthcheck:
+      test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-prootpassword"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 5
+
+  web:
+    image: git.hamburg.ccc.de/ccchh/sunders/web:latest
+    build: ./web/
+    environment:
+      MYSQL_HOST: db
+      MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}}
+      MYSQL_USER: camera_select # ${{secrets.CAMERA_SELECT_USER}}
+      MYSQL_PASSWORD: camera_selectpassword # ${{secrets.CAMERA_SELECT_USER_PASSWORD}}
+      DEFAULT_ZOOM: 12
+      DEFAULT_LAT: 0
+      DEFAULT_LON: 0
+      DEFAULT_LANGUAGE: en
+    ports:
+      - "8080:80"
+    depends_on:
+      data_handler:
+        condition: service_started
+  
+  data_handler:
+    image: git.hamburg.ccc.de/ccchh/sunders/data_handler:latest
+    build: ./data_handler/
+    environment:
+      MYSQL_HOST: db
+      MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}}
+      MYSQL_USER: root # ${{secrets.MYSQL_USER}}
+      MYSQL_PASSWORD: rootpassword # ${{secrets.MYSQL_ROOT_PASSWORD}}
+      CAMERA_USER: camera # ${{secrets.CAMERA_USER}}
+      CAMERA_USER_PASSWORD: camerapassword # ${{secrets.CAMERA_USER_PASSWORD}}
+      CAMERA_SELECT_USER: camera_select # ${{secrets.CAMERA_SELECT_USER}}
+      CAMERA_SELECT_USER_PASSWORD: camera_selectpassword # ${{secrets.CAMERA_SELECT_USER_PASSWORD}}
+    depends_on:
+      db:
+        condition: service_healthy
+        restart: true
+
+volumes:
+  mariadb:

get_rate.sh

@@ -0,0 +1,5 @@
+before=$(ls migrations | wc -l)
+sleep 60
+after=$(ls migrations | wc -l)
+let dif=after-before
+echo "$dif"

home/sunders/init_cameras/db/createDB.sql

@@ -1,7 +1,7 @@
 create database camera;
 use camera;
 
-grant all privileges on camera.* to camera@localhost identified by 'xxxxxxxx';
+grant all privileges on camera.* to camera@localhost identified by 'asdf';
 grant select on camera.* to camselect@localhost;
 
 CREATE TABLE position (

home/sunders/update_cameras/update_camera.php

@@ -15,7 +15,7 @@ $id = 0;
 $latitude = 0;
 $longitude = 0;
 
-$mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWD, MYSQL_DB);
+$mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB);
 
 if($mysqli->connect_errno) {
   echo "Error while connecting to DB : $mysqli->error \n" ;

home/sunders/update_cameras/update_statistics.php

@@ -4,7 +4,7 @@ include "config.php";
 
 if (USE_STATISTICS) {
 
-  $mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWD, MYSQL_DB);
+  $mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB);
 
   if ($mysqli->connect_errno) {
     echo "Error while connecting to DB : ".$mysqli->error." \n" ;

web/Containerfile

@@ -0,0 +1,4 @@
+FROM docker.io/library/php:apache
+
+RUN docker-php-ext-install mysqli bcmath
+COPY ./www/* /var/www/html

web/www/sunders/Leaflet/leaflet.css

@@ -45,7 +45,10 @@
 	}
 /* .leaflet-container svg: reset svg max-width decleration shipped in Joomla! (joomla.org) 3.x */
 /* .leaflet-container img: map is broken in FF if you have max-width: 100% on tiles */
-.leaflet-container .leaflet-overlay-pane svg,
+.leaflet-container .leaflet-overlay-pane svg {
+	max-width: none !important;
+	max-height: none !important;
+	}
 .leaflet-container .leaflet-marker-pane img,
 .leaflet-container .leaflet-shadow-pane img,
 .leaflet-container .leaflet-tile-pane img,
@@ -53,8 +56,15 @@
 .leaflet-container .leaflet-tile {
 	max-width: none !important;
 	max-height: none !important;
+	width: auto;
+	padding: 0;
 	}
 
+.leaflet-container img.leaflet-tile {
+	/* See: https://bugs.chromium.org/p/chromium/issues/detail?id=600120 */
+	mix-blend-mode: plus-lighter;
+}
+
 .leaflet-container.leaflet-touch-zoom {
 	-ms-touch-action: pan-x pan-y;
 	touch-action: pan-x pan-y;
@@ -166,9 +176,6 @@
 
 /* zoom and fade animations */
 
-.leaflet-fade-anim .leaflet-tile {
-	will-change: opacity;
-	}
 .leaflet-fade-anim .leaflet-popup {
 	opacity: 0;
 	-webkit-transition: opacity 0.2s linear;
@@ -183,9 +190,10 @@
 	    -ms-transform-origin: 0 0;
 	        transform-origin: 0 0;
 	}
-.leaflet-zoom-anim .leaflet-zoom-animated {
+svg.leaflet-zoom-animated {
 	will-change: transform;
-	}
+}
+
 .leaflet-zoom-anim .leaflet-zoom-animated {
 	-webkit-transition: -webkit-transform 0.25s cubic-bezier(0,0,0.25,1);
 	   -moz-transition:    -moz-transform 0.25s cubic-bezier(0,0,0.25,1);
@@ -251,14 +259,11 @@ svg.leaflet-image-layer.leaflet-interactive path {
 
 .leaflet-container {
 	background: #ddd;
-	outline: 0;
+	outline-offset: 1px;
 	}
 .leaflet-container a {
 	color: #0078A8;
 	}
-.leaflet-container a.leaflet-active {
-	outline: 2px solid orange;
-	}
 .leaflet-zoom-box {
 	border: 2px dotted #38f;
 	background: rgba(255,255,255,0.5);
@@ -267,7 +272,10 @@ svg.leaflet-image-layer.leaflet-interactive path {
 
 /* general typography */
 .leaflet-container {
-	font: 12px/1.5 "Helvetica Neue", Arial, Helvetica, sans-serif;
+	font-family: "Helvetica Neue", Arial, Helvetica, sans-serif;
+	font-size: 12px;
+	font-size: 0.75rem;
+	line-height: 1.5;
 	}
 
 
@@ -277,8 +285,7 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	box-shadow: 0 1px 5px rgba(0,0,0,0.65);
 	border-radius: 4px;
 	}
-.leaflet-bar a,
-.leaflet-bar a:hover {
+.leaflet-bar a {
 	background-color: #fff;
 	border-bottom: 1px solid #ccc;
 	width: 26px;
@@ -295,7 +302,8 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	background-repeat: no-repeat;
 	display: block;
 	}
-.leaflet-bar a:hover {
+.leaflet-bar a:hover,
+.leaflet-bar a:focus {
 	background-color: #f4f4f4;
 	}
 .leaflet-bar a:first-child {
@@ -385,6 +393,8 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	}
 .leaflet-control-layers label {
 	display: block;
+	font-size: 13px;
+	font-size: 1.08333em;
 	}
 .leaflet-control-layers-separator {
 	height: 0;
@@ -393,7 +403,7 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	}
 
 /* Default icon URLs */
-.leaflet-default-icon-path {
+.leaflet-default-icon-path { /* used only in path-guessing heuristic, see L.Icon.Default */
 	background-image: url(images/marker-icon.png);
 	}
 
@@ -402,23 +412,27 @@ svg.leaflet-image-layer.leaflet-interactive path {
 
 .leaflet-container .leaflet-control-attribution {
 	background: #fff;
-	background: rgba(255, 255, 255, 0.7);
+	background: rgba(255, 255, 255, 0.8);
 	margin: 0;
 	}
 .leaflet-control-attribution,
 .leaflet-control-scale-line {
 	padding: 0 5px;
 	color: #333;
+	line-height: 1.4;
 	}
 .leaflet-control-attribution a {
 	text-decoration: none;
 	}
-.leaflet-control-attribution a:hover {
+.leaflet-control-attribution a:hover,
+.leaflet-control-attribution a:focus {
 	text-decoration: underline;
 	}
-.leaflet-container .leaflet-control-attribution,
-.leaflet-container .leaflet-control-scale {
-	font-size: 11px;
+.leaflet-attribution-flag {
+	display: inline !important;
+	vertical-align: baseline !important;
+	width: 1em;
+	height: 0.6669em;
 	}
 .leaflet-left .leaflet-control-scale {
 	margin-left: 5px;
@@ -431,14 +445,11 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	border-top: none;
 	line-height: 1.1;
 	padding: 2px 5px 1px;
-	font-size: 11px;
 	white-space: nowrap;
-	overflow: hidden;
 	-moz-box-sizing: border-box;
 	     box-sizing: border-box;
-
-	background: #fff;
-	background: rgba(255, 255, 255, 0.5);
+	background: rgba(255, 255, 255, 0.8);
+	text-shadow: 1px 1px #fff;
 	}
 .leaflet-control-scale-line:not(:first-child) {
 	border-top: 2px solid #777;
@@ -474,17 +485,22 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	border-radius: 12px;
 	}
 .leaflet-popup-content {
-	margin: 13px 19px;
-	line-height: 1.4;
+	margin: 13px 24px 13px 20px;
+	line-height: 1.3;
+	font-size: 13px;
+	font-size: 1.08333em;
+	min-height: 1px;
 	}
 .leaflet-popup-content p {
-	margin: 18px 0;
+	margin: 17px 0;
+	margin: 1.3em 0;
 	}
 .leaflet-popup-tip-container {
 	width: 40px;
 	height: 20px;
 	position: absolute;
 	left: 50%;
+	margin-top: -1px;
 	margin-left: -20px;
 	overflow: hidden;
 	pointer-events: none;
@@ -495,6 +511,7 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	padding: 1px;
 
 	margin: -10px auto 0;
+	pointer-events: auto;
 
 	-webkit-transform: rotate(45deg);
 	   -moz-transform: rotate(45deg);
@@ -511,24 +528,21 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	position: absolute;
 	top: 0;
 	right: 0;
-	padding: 4px 4px 0 0;
 	border: none;
 	text-align: center;
-	width: 18px;
-	height: 14px;
-	font: 16px/14px Tahoma, Verdana, sans-serif;
-	color: #c3c3c3;
+	width: 24px;
+	height: 24px;
+	font: 16px/24px Tahoma, Verdana, sans-serif;
+	color: #757575;
 	text-decoration: none;
-	font-weight: bold;
 	background: transparent;
 	}
-.leaflet-container a.leaflet-popup-close-button:hover {
-	color: #999;
+.leaflet-container a.leaflet-popup-close-button:hover,
+.leaflet-container a.leaflet-popup-close-button:focus {
+	color: #585858;
 	}
 .leaflet-popup-scrolled {
 	overflow: auto;
-	border-bottom: 1px solid #ddd;
-	border-top: 1px solid #ddd;
 	}
 
 .leaflet-oldie .leaflet-popup-content-wrapper {
@@ -541,9 +555,6 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	-ms-filter: "progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678)";
 	filter: progid:DXImageTransform.Microsoft.Matrix(M11=0.70710678, M12=0.70710678, M21=-0.70710678, M22=0.70710678);
 	}
-.leaflet-oldie .leaflet-popup-tip-container {
-	margin-top: -1px;
-	}
 
 .leaflet-oldie .leaflet-control-zoom,
 .leaflet-oldie .leaflet-control-layers,
@@ -578,7 +589,7 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	pointer-events: none;
 	box-shadow: 0 1px 3px rgba(0,0,0,0.4);
 	}
-.leaflet-tooltip.leaflet-clickable {
+.leaflet-tooltip.leaflet-interactive {
 	cursor: pointer;
 	pointer-events: auto;
 	}
@@ -638,3 +649,13 @@ svg.leaflet-image-layer.leaflet-interactive path {
 	margin-left: -12px;
 	border-right-color: #fff;
 	}
+
+/* Printing */
+
+@media print {
+	/* Prevent printers from removing background-images of controls. */
+	.leaflet-control {
+		-webkit-print-color-adjust: exact;
+		print-color-adjust: exact;
+		}
+	}

web/www/sunders/add-lists.php

@@ -1,4 +1,5 @@
 <?php
+  error_reporting(0);
   // Convert the content of the symbology JSON file to HTML.
   function addListSymbology($jsonPath, $i18n, $i18nDefault) {
     global $pathToWebFolder;

web/www/sunders/camera.php

@@ -1,7 +1,7 @@
 <?php
-
+  error_reporting(0);
   include $pathToWebFolder.'config.php';
-
+  
   define('MAX_POINTS_FOR_QUICKHULL', 3000);
 
   class OsmPoint {
@@ -266,7 +266,7 @@
   $divDiag2  = ($divWidth * $divWidth) + ($divHeight * $divHeight);
 
   /* Connect to database */
-  $mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWD, MYSQL_DB);
+  $mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB);
   if($mysqli->connect_errno) {
     header('Content-type: application/json');
     $result = '{"error":"error while connecting to db : ' . $mysqli->error . '"}';

web/www/sunders/config.php

@@ -0,0 +1,15 @@
+<?php
+    define('DEFAULT_ZOOM', getenv('DEFAULT_ZOOM') ?: 11);
+    define('DEFAULT_LAT', getenv('DEFAULT_LAT') ?: 53.5550);
+    define('DEFAULT_LON', getenv('DEFAULT_LON') ?: 10.0099);
+    define('DEFAULT_LANGUAGE', getenv('DEFAULT_LANGUAGE') ?: 'en');
+    define('DEFAULT_PIE', 'country');
+    define('DEFAULT_TIME', 'single');
+
+    define('MYSQL_HOST', getenv('MYSQL_HOST') ?: '');
+    define('MYSQL_DB', getenv('MYSQL_DB') ?: '');
+    define('MYSQL_USER', getenv('MYSQL_USER') ?: '');
+    define('MYSQL_PASSWORD', getenv('MYSQL_PASSWORD') ?: '');
+
+    define('USE_STATISTICS', false);
+?>