diff --git a/.gitignore b/.gitignore index 09f94a6..1febf73 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ lastState.txt lastStatisticsUpdate.txt log.* state.txt -migrations \ No newline at end of file +migrations + +# created when running development environment +mariadb/** \ No newline at end of file diff --git a/README.md b/README.md index b2e535c..7160d32 100644 --- a/README.md +++ b/README.md @@ -35,12 +35,21 @@ To run Surveillance using docker, decide between development/testing or producti 1. Clone this repository -2. Run `docker compose up` and wait for services to start up +2. Run `docker compose -f docker-compose.yml -f docker-compose.dev.yml up` and wait for services to start up 3. Visit `http://localhost:8081/` in a browser; you should see an interactive OpenStreetMap -_tbd.: Populate camera data_ +4. You can see most cameras on the map immediately; the `data_handler` background process continuously applies updates from OpenStreetMap which become visible after refrshing the page +5. Appending `-f docker-compose.dev.yml` applies these changes: + + - PHP won't supress any errors; to disable this, remove `return;` in `./web/www/sunders/config.php` + + - Making changes in the `web` will immediately apply them to the running containers; try modifying `web/www/sunders/index.php` + + - Making changes in the `data_handler` will immediately apply them to the running containers (try modifying `data_handler/utils/update_camera.sh`) + + - The host has access to database files at `./mariadb` ### Production 1. Close this repository @@ -49,9 +58,9 @@ _tbd.: Populate camera data_ 3. Run `docker compose up` and wait for services to start up -3. Visit `http://localhost:8081/` in a browser to open the web interface +3. Visit `http://localhost:8081/` in a browser; you should see an interactive OpenStreetMap -_tbd.: Populate camera data_ +4. You can see most cameras on the map immediately; the `data_handler` background process continuously applies updates from OpenStreetMap which become visible after refrshing the page ## Surveillance nodes diff --git a/data_handler/Containerfile b/data_handler/Containerfile index 7215bf9..4c08fa6 100644 --- a/data_handler/Containerfile +++ b/data_handler/Containerfile @@ -5,4 +5,4 @@ COPY ./data_init/*.sql /opt/init/init.sql ENTRYPOINT ["/opt/entrypoint.sh"] -CMD ["/usr/sbin/dcrond", "-f" ] \ No newline at end of file +CMD ["/bin/sh", "-c", "/usr/sbin/dcrond -f -d 5 > /var/log/dcrond.log 2>&1"] \ No newline at end of file diff --git a/data_handler/utils/entrypoint.sh b/data_handler/utils/entrypoint.sh index c4dc346..9e726e8 100755 --- a/data_handler/utils/entrypoint.sh +++ b/data_handler/utils/entrypoint.sh @@ -8,7 +8,7 @@ set -e cat<> /var/log/update_camera_$(date +\%Y\%m\%d\%H\%M\%S).log 2>&1 EOF #################################################################################################### @@ -20,7 +20,15 @@ php /opt/init_db.php echo "Prescripts done" #################################################################################################### -### Run cmd +### Run script once immediately +#################################################################################################### + +echo "Running update once initially..." +sh /opt/update_camera.sh >> /var/log/update_camera_$(date +\%Y\%m\%d\%H\%M\%S).log 2>&1 +echo "Initial update done" + +#################################################################################################### +### Run cron to schedule periodic task #################################################################################################### # see: https://github.com/dubiousjim/dcron/issues/13 diff --git a/data_handler/utils/update_camera.sh b/data_handler/utils/update_camera.sh index 25d23c8..4107d9f 100644 --- a/data_handler/utils/update_camera.sh +++ b/data_handler/utils/update_camera.sh @@ -63,7 +63,7 @@ do rm "change_file.osc.gz" fi - echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName" + echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum of $newSeqNum" > "$logFileName" targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'` targetDirName="$REPLICATE_URL/$targetDirName" @@ -87,7 +87,7 @@ do targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'` curl -L "$targetDirName" -o lastState.txt 2>&1 | tee -a $logFileName - echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName" + echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum of $newSeqNum" | tee -a "$logFileName" done rm "/var/lock/update_camera" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..61d0016 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,15 @@ +services: + db: + volumes: + - ./mariadb:/var/lib/mysql:Z + + web: + volumes: + - ./web/www/sunders:/var/www/html:Z + environment: + APP_ENV: development + + data_handler: + volumes: + - ./data_handler/utils:/opt:Z + - ./data_handler/data_init:/opt/init/init.sql:Z \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 8954b73..99ed328 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: db: image: mariadb:13.0.2 + restart: unless-stopped command: --max_allowed_packet=3250585600 environment: MYSQL_ROOT_PASSWORD: rootpassword # ${{secrets.MYSQL_ROOT_PASSWORD}} @@ -10,7 +11,7 @@ services: volumes: - mariadb:/var/lib/mysql healthcheck: - test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-prootpassword"] + test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-prootpassword"] # ${{secrets.MYSQL_ROOT_PASSWORD}} interval: 10s timeout: 5s start_period: 30s @@ -18,6 +19,7 @@ services: web: image: git.hamburg.ccc.de/ccchh/sunders/web:latest + restart: unless-stopped environment: MYSQL_HOST: db MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}} @@ -31,11 +33,18 @@ services: ports: - "8081:80" depends_on: - data_handler: + db: condition: service_started + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost/sync-state.php"] + interval: 10s + timeout: 5s + start_period: 30s + retries: 5 data_handler: image: git.hamburg.ccc.de/ccchh/sunders/data_handler:latest + restart: unless-stopped environment: MYSQL_HOST: db MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}} @@ -49,6 +58,13 @@ services: db: condition: service_healthy restart: true + # if the latest `update_camera_*.log` contains "error", fail the healthcheck + healthcheck: + test: ["CMD", "sh", "-c", "ls /var/log | grep '^update_camera_' | xargs -I {} sh -c 'grep -q error /var/log/{} || exit 0; exit 1'"] + interval: 10s + timeout: 5s + start_period: 30s + retries: 5 volumes: mariadb: \ No newline at end of file diff --git a/home/sunders/update_cameras/update_camera.sh b/home/sunders/update_cameras/update_camera.sh index b605ee4..b6d3338 100644 --- a/home/sunders/update_cameras/update_camera.sh +++ b/home/sunders/update_cameras/update_camera.sh @@ -60,7 +60,7 @@ do rm "change_file.osc.gz" fi - echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName" + echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum of $newSeqNum" > "$logFileName" targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'` targetDirName="$REPLICATE_URL/$targetDirName" @@ -84,7 +84,7 @@ do targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'` wget -nv "$targetDirName" -O lastState.txt 2>&1 | tee -a $logFileName - echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName" + echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum of $newSeqNum" | tee -a "$logFileName" done rm "/var/lock/update_camera" diff --git a/web/www/sunders/add-lists.php b/web/www/sunders/add-lists.php index da9f396..12fc0c0 100644 --- a/web/www/sunders/add-lists.php +++ b/web/www/sunders/add-lists.php @@ -1,5 +1,4 @@ \ No newline at end of file diff --git a/web/www/sunders/sync-state.php b/web/www/sunders/sync-state.php new file mode 100644 index 0000000..3cb8ef0 --- /dev/null +++ b/web/www/sunders/sync-state.php @@ -0,0 +1,30 @@ +connect_errno) { + throw new Exception("Could not connect to database"); + } + + $syncstate_querry = $mysqli->query("SELECT * FROM sync_state WHERE k = 'sequenceNumber'"); + + while($row = $syncstate_querry->fetch_assoc()) { + $syncstate = array('sequenceNumber' => $row["v"]); + } + + $result = json_encode($syncstate); + + $mysqli->close(); + + header('Content-type: application/json; Charset : utf-8'); + echo $result; + mysqli_report(MYSQLI_REPORT_OFF); + } catch (Exception $e) { + http_response_code(503); + header('Content-type: application/json'); + $result = '{"error":"Database unavailable"}'; + echo $result; + } +?>