Merge pull request 'Adds health checks to web and data_handler container; restarting on failure' (#24) from healthcheck into main

Reviewed-on: #24
Reviewed-by: stb <stb@noreply.git.hamburg.ccc.de>

.gitignore

@@ -3,4 +3,7 @@ lastState.txt
 lastStatisticsUpdate.txt
 log.*
 state.txt
-migrations
+migrations
+
+# created when running development environment
+mariadb/**

README.md

@@ -35,12 +35,21 @@ To run Surveillance using docker, decide between development/testing or producti
 
 1. Clone this repository
 
-2. Run `docker compose up` and wait for services to start up
+2. Run `docker compose -f docker-compose.yml -f docker-compose.dev.yml up` and wait for services to start up
 
 3. Visit `http://localhost:8081/` in a browser; you should see an interactive OpenStreetMap
 
-_tbd.: Populate camera data_
+4. You can see most cameras on the map immediately; the `data_handler` background process continuously applies updates from OpenStreetMap which become visible after refrshing the page
 
+5. Appending `-f docker-compose.dev.yml` applies these changes:
+
+   - PHP won't supress any errors; to disable this, remove `return;` in `./web/www/sunders/config.php`
+
+   - Making changes in the `web` will immediately apply them to the running containers; try modifying `web/www/sunders/index.php`
+
+   - Making changes in the `data_handler` will immediately apply them to the running containers (try modifying `data_handler/utils/update_camera.sh`)
+
+   - The host has access to database files at `./mariadb`
 ### Production
 
 1. Close this repository
@@ -51,7 +60,7 @@ _tbd.: Populate camera data_
 
 3. Visit `http://localhost:8081/` in a browser to open the web interface
 
-_tbd.: Populate camera data_
+4. You can see most cameras on the map immediately; the `data_handler` background process continuously applies updates from OpenStreetMap which become visible after refrshing the page
 
 ## Surveillance nodes
 

data_handler/Containerfile

@@ -5,4 +5,4 @@ COPY ./data_init/*.sql /opt/init/init.sql
 
 ENTRYPOINT ["/opt/entrypoint.sh"]
 
-CMD ["/usr/sbin/dcrond", "-f" ]
+CMD ["/usr/sbin/dcrond", "-f", "-d", "5"]

data_handler/utils/entrypoint.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 set -e
 
+printf 'ok\n' > /run/update_camera.status
 
 ####################################################################################################
 ### Crontab settings
@@ -8,7 +9,7 @@ set -e
 
 cat<<EOF | dcrontab -
 # min     hour      day     month     weekday command
-5    *   *       *         *       sh /opt/update_camera.sh
+*/15   *   *       *         *       sh /opt/run_update_camera.sh
 EOF
 
 ####################################################################################################
@@ -20,7 +21,15 @@ php /opt/init_db.php
 echo "Prescripts done"
 
 ####################################################################################################
-### Run cmd
+### Run script once immediately
+####################################################################################################
+
+echo "Running update once initially..."
+sh /opt/run_update_camera.sh
+echo "Initial update done"
+
+####################################################################################################
+### Run cron to schedule periodic task
 ####################################################################################################
 
 # see: https://github.com/dubiousjim/dcron/issues/13

data_handler/utils/run_update_camera.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+status_file=/run/update_camera.status
+status_file_tmp="${status_file}.$$"
+
+if sh /opt/update_camera.sh 2>&1 | tee /proc/1/fd/1 | grep -i error >/dev/null; then
+    printf 'error\n' > "$status_file_tmp"
+else
+    printf 'ok\n' > "$status_file_tmp"
+fi
+
+mv "$status_file_tmp" "$status_file"

data_handler/utils/update_camera.sh

@@ -63,7 +63,7 @@ do
     rm "change_file.osc.gz"
   fi
 
-  echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName"
+  echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum of $newSeqNum" > "$logFileName"
 
   targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'`
   targetDirName="$REPLICATE_URL/$targetDirName"
@@ -87,7 +87,7 @@ do
   targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'`
 
   curl -L "$targetDirName" -o lastState.txt 2>&1 | tee -a $logFileName
-  echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName"
+  echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum of $newSeqNum" | tee -a "$logFileName"
 done
 
 rm "/var/lock/update_camera"

docker-compose.dev.yml

@@ -0,0 +1,15 @@
+services:
+  db:
+    volumes:
+        - ./mariadb:/var/lib/mysql:Z
+
+  web:
+    volumes:
+      - ./web/www/sunders:/var/www/html:Z
+    environment:
+      APP_ENV: development
+
+  data_handler:
+    volumes:
+      - ./data_handler/utils:/opt:Z
+      - ./data_handler/data_init:/opt/init/init.sql:Z

docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   db:
     image: mariadb:13.0.2
+    restart: unless-stopped
     command: --max_allowed_packet=3250585600
     environment:
       MYSQL_ROOT_PASSWORD: rootpassword # ${{secrets.MYSQL_ROOT_PASSWORD}}
@@ -10,7 +11,7 @@ services:
     volumes:
         - mariadb:/var/lib/mysql
     healthcheck:
-      test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-prootpassword"]
+      test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-prootpassword"] # ${{secrets.MYSQL_ROOT_PASSWORD}}
       interval: 10s
       timeout: 5s
       start_period: 30s
@@ -18,6 +19,7 @@ services:
 
   web:
     image: git.hamburg.ccc.de/ccchh/sunders/web:latest
+    restart: unless-stopped
     environment:
       MYSQL_HOST: db
       MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}}
@@ -29,13 +31,20 @@ services:
       DEFAULT_LANGUAGE: en
       IMPRESSUM_URL: https://hamburg.ccc.de/imprint/
     ports:
-      - "8081:80"
+      - "8080:80"
     depends_on:
-      data_handler:
+      db:
         condition: service_started
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost/sync-state.php"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 5
   
   data_handler:
     image: git.hamburg.ccc.de/ccchh/sunders/data_handler:latest
+    restart: unless-stopped
     environment:
       MYSQL_HOST: db
       MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}}
@@ -49,6 +58,13 @@ services:
       db:
         condition: service_healthy
         restart: true
+    # if the latest update output contains "error", fail the healthcheck
+    healthcheck:
+      test: ["CMD", "sh", "-c", "test \"$(cat /run/update_camera.status)\" = ok"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 5
 
 volumes:
   mariadb:

home/sunders/update_cameras/update_camera.sh

@@ -60,7 +60,7 @@ do
     rm "change_file.osc.gz"
   fi
 
-  echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName"
+  echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum of $newSeqNum" > "$logFileName"
 
   targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'`
   targetDirName="$REPLICATE_URL/$targetDirName"
@@ -84,7 +84,7 @@ do
   targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'`
   wget -nv "$targetDirName" -O lastState.txt 2>&1 | tee -a $logFileName
 
-  echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName"
+  echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum of $newSeqNum" | tee -a "$logFileName"
 done
 
 rm "/var/lock/update_camera"

web/Containerfile

@@ -1,4 +1,4 @@
-FROM docker.io/library/php:apache
+FROM docker.io/library/php:8.5.12-apache
 
 RUN docker-php-ext-install mysqli bcmath
 

web/www/sunders/add-lists.php

@@ -1,5 +1,4 @@
 <?php
-  error_reporting(0);
   // Convert the content of the symbology JSON file to HTML.
   function addListSymbology($jsonPath, $i18n, $i18nDefault) {
     global $pathToWebFolder;

web/www/sunders/camera.php

@@ -1,5 +1,4 @@
 <?php
-  error_reporting(0);
   include $pathToWebFolder.'config.php';
   
   define('MAX_POINTS_FOR_QUICKHULL', 3000);

web/www/sunders/config.php

@@ -13,4 +13,15 @@
     define('MYSQL_PASSWORD', getenv('CAMERA_SELECT_USER_PASSWORD') ?: '');
 
     define('USE_STATISTICS', false);
+
+
+    $environment = getenv('APP_ENV') ?: 'production';
+    if ($environment === 'development') {
+        ini_set('display_errors', 1);
+        error_reporting(E_ALL);
+        return;
+    }
+     
+    ini_set('display_errors', 0);
+    error_reporting(0);
 ?>

web/www/sunders/sync-state.php

@@ -0,0 +1,30 @@
+<?php
+  error_reporting(0);
+  include $pathToWebFolder.'config.php';
+
+  try {
+    $mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB);
+    if($mysqli->connect_errno) {
+      throw new Exception("Could not connect to database");
+    }
+
+    $syncstate_querry = $mysqli->query("SELECT * FROM sync_state WHERE k = 'sequenceNumber'");
+
+    while($row = $syncstate_querry->fetch_assoc()) {
+        $syncstate = array('sequenceNumber' => $row["v"]);
+      }
+
+    $result = json_encode($syncstate);
+
+    $mysqli->close();
+
+    header('Content-type: application/json; Charset : utf-8');
+    echo $result;
+    mysqli_report(MYSQLI_REPORT_OFF);
+  } catch (Exception $e) {
+      http_response_code(503);
+      header('Content-type: application/json');
+      $result = '{"error":"Database unavailable"}';
+      echo $result;
+  }
+?>