From 3590da7e269e4996410d3fd4c74bda17a7c4d155 Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Wed, 17 Jun 2026 16:50:13 +0200 Subject: [PATCH 01/30] fix: Removes offline and unavailable links --- web/www/sunders/json/links.json | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/web/www/sunders/json/links.json b/web/www/sunders/json/links.json index f597479..d7cd3cc 100644 --- a/web/www/sunders/json/links.json +++ b/web/www/sunders/json/links.json @@ -3,22 +3,17 @@ { "sectionTitle": "", "isExpandable": false, "isExpandedByDefault": true, "listEntries": [ { "sourceText": "1984 Actionday", "href": "https://1984actionday.wordpress.com/", "linkText": "When will you fight back?" }, { "sourceText": "ACLU", "href": "https://www.aclu.org/sites/default/files/field_document/061819-robot_surveillance.pdf", "linkText": "The Dawn of Robot Surveillance" }, - { "sourceText": "Adversarial Fashion", "href": "https://adversarialfashion.com/", "linkText": "Patterns on goods designed to trigger Automated License Plate Readers" }, { "sourceText": "Atlas of Surveillance", "href": "https://atlasofsurveillance.org/", "linkText": "Documenting Police Tech in Our Communities" }, { "sourceText": "Ban Facial Recognition", "href": "https://www.banfacialrecognition.com/", "linkText": "Ban Facial Recognition" }, { "sourceText": "Ban Facial Recognition", "href": "https://www.banfacialrecognition.com/festivals/", "linkText": "Ban Facial Recognition at Festivals" }, - { "sourceText": "Ban Facial Recognition", "href": "https://ban-facial-recognition.wesign.it/droitshumains/ban-facial-recognition-europe", "linkText": "Ban Facial Recognition Europe!" }, { "sourceText": "Electronic Frontier Foundation", "href": "https://www.eff.org/sls", "linkText": "Street Level Surveillance" }, - { "sourceText": "Every Two Years", "href": "https://everytwoyears.org/", "linkText": "Help Make Democracy" }, { "sourceText": "Fawkes", "href": "http://sandlab.cs.uchicago.edu/fawkes/", "linkText": "Image “Cloaking” for Personal Privacy" }, { "sourceText": "GitHub", "href": "https://github.com/unsurv", "linkText": "unsurv" }, - { "sourceText": "Insecam", "href": "https://www.insecam.org/", "linkText": "World biggest online cameras directory" }, + { "sourceText": "Insecam", "href": "http://www.insecam.org/", "linkText": "World biggest online cameras directory" }, { "sourceText": "No CCTV", "href": "http://www.no-cctv.org.uk/", "linkText": "Campaigning against camera surveillance in the UK & beyond" }, { "sourceText": "NYC Surveillance Camera Project", "href": "http://www.mediaeater.com/cameras/", "linkText": "NYC Surveillance Camera Project" }, { "sourceText": "Reclaim Your Face", "href": "https://reclaimyourface.eu/", "linkText": "Reclaim Your Face" }, { "sourceText": "S.T.O.P.", "href": "https://www.stopspying.org/", "linkText": "The Surveillance Technology Oversight Project" }, - { "sourceText": "Spy Blog", "href": "https://p10.secure.hostingprod.com/@spyblog.org.uk/ssl/spyblog/cctv-surveillance-cameras/", "linkText": "Category: CCTV surveillance cameras" }, - { "sourceText": "unsurv", "href": "https://www.unsurv.org/", "linkText": "unsurv.org" }, { "sourceText": "Wikipedia", "href": "https://www.wikipedia.org/wiki/Mass_surveillance_in_the_United_States", "linkText": "Mass Surveillance in the United States" }, { "sourceText": "Yale University", "href": "https://privacylab.yale.edu/", "linkText": "Privacy Lab" } ] } @@ -27,7 +22,6 @@ { "listTitle": "fra", "sections": [ { "sectionTitle": "", "isExpandable": false, "isExpandedByDefault": true, "listEntries": [ - { "sourceText": "Ban Facial Recognition", "href": "https://ban-facial-recognition.wesign.it/droitshumains/bannissons-la-reconnaissance-faciale-en-europe", "linkText": "Bannissons la reconnaissance faciale en Europe!" }, { "sourceText": "BUG BROTHER", "href": "http://bugbrother.blog.lemonde.fr/", "linkText": "Qui surveillera les surveillants?" }, { "sourceText": "LA QUADRATURE DU NET", "href": "https://www.laquadrature.net/fr", "linkText": "Internet & Libertés" }, { "sourceText": "Reclaim Your Face", "href": "https://reclaimyourface.eu/fr/", "linkText": "Reclaim Your Face FR" }, @@ -38,20 +32,17 @@ { "listTitle": "ger", "sections": [ { "sectionTitle": "", "isExpandable": false, "isExpandedByDefault": true, "listEntries": [ - { "sourceText": "AK Vorrat", "href": "http://wiki.vorratsdatenspeicherung.de/Arbeitsgruppe_Video%C3%BCberwachung", "linkText": "Arbeitsgruppe Videoüberwachung" }, + { "sourceText": "AK Vorrat", "href": "http://wiki.vorratsdatenspeicherung.de/Arbeitsgruppe_Videoüberwachung", "linkText": "Arbeitsgruppe Videoüberwachung" }, { "sourceText": "AK Vorrat", "href": "http://wiki.vorratsdatenspeicherung.de/images/Folder_cctv.pdf", "linkText": "Infobroschüre: Videoüberwachung? Sicherheit geht anders!" }, { "sourceText": "AK Vorrat, Ortsgruppe Hannover", "href": "http://wiki.vorratsdatenspeicherung.de/Ortsgruppen/Hannover/Videoueberwachung_in_Hannover", "linkText": "Videoüberwachung in Hannover" }, { "sourceText": "AKTION FREIHEIT STATT ANGST", "href": "https://www.aktion-freiheitstattangst.org/", "linkText": "Aktion Freiheit statt Angst e.V." }, - { "sourceText": "Ban Facial Recognition", "href": "https://ban-facial-recognition.wesign.it/droitshumains/verbot-der-gesichtserkennung-in-europa", "linkText": "Verbot der Gesichtserkennung in Europa!" }, { "sourceText": "Benjamin J. Kees", "href": "https://algoropticon.de/", "linkText": "Algorithmisches Panopticon" }, { "sourceText": "dieDatenschützer Rhein-Main", "href": "https://ddrm.de/category/videouberwachung-2/", "linkText": "Kategoriearchiv: Videoüberwachung" }, { "sourceText": "EPICENTER.WORKS", "href": "https://epicenter.works/thema/face-recognition", "linkText": "Face Recognition - Gesichtserkennung" }, { "sourceText": "EPICENTER.WORKS", "href": "https://überwachungspaket.at/", "linkText": "Stoppt das Überwachungspaket!" }, { "sourceText": "Frag den Staat", "href": "https://fragdenstaat.de/anfrage/informationen-zum-projekt-sicherheitsbahnhof-berlin-sudkreuz/", "linkText": "Informationen zum „Projekt Sicherheitsbahnhof“ Berlin Südkreuz" }, { "sourceText": "Frag den Staat", "href": "https://fragdenstaat.de/anfrage/datenschutzkonzept-fur-videouberwachung-am-bahnhof-berlin-sudkreuz/", "linkText": "Datenschutzkonzept für Videoüberwachung am Bahnhof Berlin Südkreuz" }, - { "sourceText": "Kassel-Watch-Log", "href": "https://www.ks-watch.de/", "linkText": "Videoüberwachung im öffentlichen Raum Kassel" }, { "sourceText": "Reclaim Your Face", "href": "https://reclaimyourface.eu/de/", "linkText": "Reclaim Your Face DE" }, - { "sourceText": "_STATTKAMERAS", "href": "https://stattkameras.de/", "linkText": "Gegen Videoüberwachung am Hansaplatz" } ] } ] } From c6d87997b00cad5cf3375bfd3340107defdc00d5 Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Wed, 17 Jun 2026 16:56:15 +0200 Subject: [PATCH 02/30] fix: Removes buildx --- .forgejo/workflows/images.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.forgejo/workflows/images.yml b/.forgejo/workflows/images.yml index 6dfac7b..190d8d4 100644 --- a/.forgejo/workflows/images.yml +++ b/.forgejo/workflows/images.yml @@ -11,9 +11,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Container Registry uses: docker/login-action@v3 with: From f407b077f28a4f4d44b552b05a7c2efccd85ab01 Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Wed, 17 Jun 2026 16:59:21 +0200 Subject: [PATCH 03/30] fix: Uses docker binary --- .forgejo/workflows/images.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.forgejo/workflows/images.yml b/.forgejo/workflows/images.yml index 190d8d4..bc48a98 100644 --- a/.forgejo/workflows/images.yml +++ b/.forgejo/workflows/images.yml @@ -11,12 +11,25 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Check Docker CLI and daemon + run: | + if ! command -v docker >/dev/null 2>&1; then + echo "::error::docker CLI not found in PATH on this runner" + exit 1 + fi + + if ! docker info >/dev/null 2>&1; then + echo "::error::docker daemon is not reachable (check /var/run/docker.sock or DOCKER_HOST)" + docker version || true + env | grep '^DOCKER' || true + ls -l /var/run/docker.sock || true + exit 1 + fi + - name: Login to Container Registry - uses: docker/login-action@v3 - with: - registry: git.hamburg.ccc.de - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_TOKEN }} + if: github.event_name == 'push' + run: | + echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.hamburg.ccc.de -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin - name: Set image tags id: vars From b2544cf2678875b108c82c6c10ced3458bc1c69d Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Wed, 17 Jun 2026 17:02:05 +0200 Subject: [PATCH 04/30] refactor: Uses kaniko builder --- .forgejo/workflows/images.yml | 68 ++++++++++++++--------------------- 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/.forgejo/workflows/images.yml b/.forgejo/workflows/images.yml index bc48a98..cb999c5 100644 --- a/.forgejo/workflows/images.yml +++ b/.forgejo/workflows/images.yml @@ -7,61 +7,47 @@ on: jobs: build: runs-on: docker + container: + image: ghcr.io/osscontainertools/kaniko:alpine steps: + - name: Install required system packages + run: apk add --no-cache nodejs git + - name: Checkout repository uses: actions/checkout@v4 - - - name: Check Docker CLI and daemon - run: | - if ! command -v docker >/dev/null 2>&1; then - echo "::error::docker CLI not found in PATH on this runner" - exit 1 - fi - - if ! docker info >/dev/null 2>&1; then - echo "::error::docker daemon is not reachable (check /var/run/docker.sock or DOCKER_HOST)" - docker version || true - env | grep '^DOCKER' || true - ls -l /var/run/docker.sock || true - exit 1 - fi - - - name: Login to Container Registry - if: github.event_name == 'push' - run: | - echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.hamburg.ccc.de -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin - name: Set image tags id: vars run: | - if [ "${{ github.ref_name }}" = "main" ] && [ "${{ github.event_name }}" = "push" ]; then + if [ "${{ forgejo.ref_name }}" = "main" ] && [ "${{ forgejo.event_name }}" = "push" ]; then echo "tag=latest" >> $GITHUB_OUTPUT else # renovate creates sub branches with `/`; these break the tagging in the build process - echo tag=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') >> $GITHUB_OUTPUT + echo tag=$(echo "${{ forgejo.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') >> $GITHUB_OUTPUT fi - name: Build web image + env: + KANIKO_NO_PUSH: ${{ forgejo.event_name != 'push' }} + KANIKO_GIT_HAMBURG_CCC_DE_USER: ${{ secrets.REGISTRY_USERNAME }} + KANIKO_GIT_HAMBURG_CCC_DE_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} run: | - docker build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} . - working-directory: ./web + /kaniko/executor \ + --dockerfile="${{ forgejo.workspace }}/web/Containerfile" \ + --context="dir://${{ forgejo.workspace }}/web" \ + --destination=git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} \ + --no-push-cache \ + --credential-helpers=env - name: Build data_handler image + env: + KANIKO_NO_PUSH: ${{ forgejo.event_name != 'push' }} + KANIKO_GIT_HAMBURG_CCC_DE_USER: ${{ secrets.REGISTRY_USERNAME }} + KANIKO_GIT_HAMBURG_CCC_DE_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} run: | - docker build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} . - working-directory: ./data_handler - - - name: Push images to Container Registry - run: | - docker push git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} - docker push git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} - if: github.event_name == 'push' - - - name: Update docker-compose.yml image tags - run: | - sed -i "s/:latest/:${{ steps.vars.outputs.tag }}/g" docker-compose.yml - - - name: Start Docker Compose services - run: | - docker compose up -d --wait - docker compose down \ No newline at end of file + /kaniko/executor \ + --dockerfile="${{ forgejo.workspace }}/data_handler/Containerfile" \ + --context="dir://${{ forgejo.workspace }}/data_handler" \ + --destination=git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} \ + --no-push-cache \ + --credential-helpers=env \ No newline at end of file From 100762ea2e2e3b0236edaea1782b55248d547b7e Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:36:49 +0100 Subject: [PATCH 05/30] fix: Escapes OSM tag data in camera popups camera.php built its JSON by hand with htmlentities(), which leaves backslashes intact, and the client parsed it with eval() and inserted tag values into the popup as HTML. A tag value such as therefore became live markup (stored XSS for anyone able to edit OSM data), and a lone backslash broke parsing of the whole response. - camera.php now builds the response with json_encode() and returns raw values - tags named lat, lon, id, count, multi or poly no longer override the fields set by the server - error responses no longer include database error messages - the client uses JSON.parse() and escapes all keys and values; only http(s) URLs are rendered as links/images --- web/www/sunders/camera.php | 123 ++++++++----------- web/www/sunders/js/leafletembed_functions.js | 39 ++++-- 2 files changed, 79 insertions(+), 83 deletions(-) diff --git a/web/www/sunders/camera.php b/web/www/sunders/camera.php index 97a77e7..9aa2f85 100644 --- a/web/www/sunders/camera.php +++ b/web/www/sunders/camera.php @@ -4,6 +4,17 @@ define('MAX_POINTS_FOR_QUICKHULL', 3000); + // Fields set by this script; OSM tags with these keys are not passed on. + define('RESERVED_KEYS', array('lat', 'lon', 'id', 'count', 'multi', 'poly')); + + define('JSON_FLAGS', JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE); + + function exitWithError($message) { + header('Content-type: application/json; charset=utf-8'); + echo json_encode(array('error' => $message), JSON_FLAGS); + exit; + } + class OsmPoint { var $id; var $lat; @@ -179,63 +190,47 @@ || !array_key_exists('zoom', $_GET) || !array_key_exists('width', $_GET) || !array_key_exists('height', $_GET)) { - header('Content-type: application/json'); - $result = '{"error":"bbox, zoom, width and height parameters are required. ' + exitWithError('bbox, zoom, width and height parameters are required. ' .(array_key_exists('bbox', $_GET) ? '' : 'bbox is empty. ') .(array_key_exists('zoom', $_GET) ? '' : 'zoom is empty. ') .(array_key_exists('width', $_GET) ? '' : 'width is empty. ') - .(array_key_exists('height', $_GET) ? '' : 'height is empty. ') - .'"}'; - echo $result; - exit; + .(array_key_exists('height', $_GET) ? '' : 'height is empty. ')); } /* Check zoom */ $zoom = $_GET['zoom']; - if ( !is_numeric($zoom) + if ( !is_string($zoom) + || !is_numeric($zoom) || intval($zoom) < 1 || intval($zoom) > 19) { - header('Content-type: application/json'); - $result = '{"error":"unexpected zoom value : ' - .htmlentities($zoom) - .'"}'; - echo $result; - exit; + exitWithError('unexpected zoom value'); } /* Check bounds box --> bbox=boundsSW.lng,boundsSW.lat,boundsNE.lng,boundsNE.lat */ + if (!is_string($_GET['bbox'])) { + exitWithError('unexpected bbox value'); + } $bbox = explode(',', $_GET['bbox']); - if ( !is_numeric($bbox[0]) || $bbox[0] > 180 // 0 = SW.lng + if ( count($bbox) != 4 + || !is_numeric($bbox[0]) || $bbox[0] > 180 // 0 = SW.lng || !is_numeric($bbox[2]) || $bbox[2] < -180 // 2 = NE.lng || !is_numeric($bbox[1]) || $bbox[1] < -90 || $bbox[1] > 90 // 1 = SW.lat || !is_numeric($bbox[3]) || $bbox[3] < -90 || $bbox[3] > 90) { // 3 = NE.lat - header('Content-type: application/json'); - $result = '{"error":"unexpected bbox longitude and latitude values : ' - .'lon ['.htmlentities($bbox[0]).', '.htmlentities($bbox[2]).'], ' - .'lat ['.htmlentities($bbox[1]).', '.htmlentities($bbox[3]).']' - .'"}'; - echo $result; - exit; + exitWithError('unexpected bbox longitude and latitude values : ' + .'lon ['.$bbox[0].', '.$bbox[2].'], ' + .'lat ['.$bbox[1].', '.$bbox[3].']'); } if ($bbox[0] >= $bbox[2]) { // 0 = SW.lng | 2 = NE.lng - header('Content-type: application/json'); - $result = '{"error":"min longitude greater than max longitude : ' - .'lon ['.htmlentities($bbox[0]).', '.htmlentities($bbox[2]).']' - .'"}'; - echo $result; - exit; + exitWithError('min longitude greater than max longitude : ' + .'lon ['.$bbox[0].', '.$bbox[2].']'); } if ($bbox[1] >= $bbox[3]) { // 1 = SW.lat | 3 = NE.lat - header('Content-type: application/json'); - $result = '{"error":"min latitude greater than max latitude : ' - .'lat ['.htmlentities($bbox[1]).', '.htmlentities($bbox[3]).']' - .'"}'; - echo $result; - exit; + exitWithError('min latitude greater than max latitude : ' + .'lat ['.$bbox[1].', '.$bbox[3].']'); } $lonMin = $bbox[0]; // 0 = SW.lng @@ -247,12 +242,7 @@ $pixelHeight = (int) $_GET['height']; if ($pixelWidth == 0 || $pixelHeight == 0) { - header('Content-type: application/json'); - $result = '{"error":"width or height is null : ' - .htmlentities($pixelWidth).'x'.htmlentities($pixelHeight) - .'"}'; - echo $result; - exit; + exitWithError('width or height is null : '.$pixelWidth.'x'.$pixelHeight); } $lonWidth = $lonMax - $lonMin; // NE.lng - SW.lng @@ -268,10 +258,7 @@ /* Connect to database */ $mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB); if($mysqli->connect_errno) { - header('Content-type: application/json'); - $result = '{"error":"error while connecting to db : ' . $mysqli->error . '"}'; - echo $result; - exit; + exitWithError('error while connecting to db'); } $rqtLonMin = $lonMin; // SW.lng @@ -365,10 +352,7 @@ $stmt->close(); } else { $mysqli->close(); - header('Content-type: application/json'); - $result = '{"error":"Error while request : ' . $mysqli->error . '"}'; - echo $result; - exit; + exitWithError('error while querying db'); } /* Unify some clusters if nodes center are near */ @@ -380,8 +364,7 @@ } } - $result='['; - $separator=''; + $result = array(); /* Writing selected points */ $sql='SELECT k, v @@ -393,53 +376,49 @@ $stmt->bind_param('d', $id); $stmt->bind_result($k, $v); - /* Writing grouped items or not */ + /* Writing grouped items or not. All values are passed on unescaped; the client + must escape them before inserting them into HTML. */ foreach($resArray as $val) { if ($val['count'] > 0) { - $result = $result.$separator - .'{"lat":"'.($val['latitude'] / $val['count']).'"' - .',"lon":"'.($val['longitude'] / $val['count']).'"'; + $entry = array( + 'lat' => (string) ($val['latitude'] / $val['count']), + 'lon' => (string) ($val['longitude'] / $val['count'])); if ($val['count'] == 1) { $id = $val['points'][0]->id; - $result = $result.',"id":"'.$id.'"'; + $entry['id'] = (string) $id; $stmt->execute(); while($stmt->fetch()) { - $result = $result.',"'.htmlentities($k).'":"'.htmlentities($v, ENT_COMPAT, 'UTF-8').'"'; + if (!in_array($k, RESERVED_KEYS, true)) { + $entry[$k] = (string) $v; + } } } else { - $result = $result - .',"count":"'.$val['count'].'"' - .',"multi":"yes"' - .',"poly":['; + $entry['count'] = (string) $val['count']; + $entry['multi'] = 'yes'; + $entry['poly'] = array(); if ($pointsCount < MAX_POINTS_FOR_QUICKHULL) { $convexPoly = quickHull($val['points'], $val['count']); - $sepPoly = ''; foreach($convexPoly as $point) { - $result = $result.$sepPoly.'{' - .'"lat":"'.$point->lat.'"' - .',"id":"'.$point->id.'"' - .',"lon":"'.$point->lon.'"}'; - $sepPoly=','; + $entry['poly'][] = array( + 'lat' => (string) $point->lat, + 'id' => (string) $point->id, + 'lon' => (string) $point->lon); } } - $result=$result.']'; } - $result = $result.'}'; - $separator=','; + $result[] = $entry; } } $stmt->close(); - $result = $result . ']'; - $mysqli->close(); - header('Content-type: application/json; Charset : utf-8'); - echo $result; + header('Content-type: application/json; charset=utf-8'); + echo json_encode($result, JSON_FLAGS); ?> diff --git a/web/www/sunders/js/leafletembed_functions.js b/web/www/sunders/js/leafletembed_functions.js index cb199a0..75e461e 100644 --- a/web/www/sunders/js/leafletembed_functions.js +++ b/web/www/sunders/js/leafletembed_functions.js @@ -129,7 +129,11 @@ function onClick(e) { function onStateChanged() { if (ajaxRequest.readyState == 4) { // 0 = UNSENT, 1 = OPENED, 2 = HEADERS_RECEIVED, 3 = LOADING, 4 = DONE if (ajaxRequest.status == 200) { // 200 = OK, 404 = Page not found - plotList = eval('(' + ajaxRequest.responseText + ')'); + try { + plotList = JSON.parse(ajaxRequest.responseText); + } catch(e) { + return; + } removeMarkers(); for (i=0; i/g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + // Add camera popup to camera marker. +// All values in plot come unescaped from OSM data and must be escaped. function addCameraDetailsData(plotMarker, plot) { + var id = escapeHtml(plot.id); popupDataTable = '' - + '' - // + '' - + '' - + ''; + + '' + // + '' + + '' + + ''; for (x in plot) { if (plot[x] !== '' && x != 'multi' && x != 'id' && x != 'userid' && x != 'lat' && x != 'lon') { - popupDataTable = popupDataTable + ''; } From 6a640831753f0dacc4826567a7c4d6794fd046cc Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:36:49 +0100 Subject: [PATCH 06/30] fix: Escapes location search results Nominatim results were inserted via innerHTML and loaded as a JSONP script with an unencoded query. Results are now fetched as JSON, the query is URL encoded, text is escaped and coordinates must be numeric. --- web/www/sunders/js/search.js | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/web/www/sunders/js/search.js b/web/www/sunders/js/search.js index 4ba00a0..38f69a5 100644 --- a/web/www/sunders/js/search.js +++ b/web/www/sunders/js/search.js @@ -37,15 +37,32 @@ function searchLocation(maxResults) { searchLimit = 11; } - var searchScript = document.createElement('script'); - searchScript.src = 'https://nominatim.openstreetmap.org/search.php?q=' + searchString + '&format=json&json_callback=parseResult&limit=' + searchLimit; - document.body.appendChild(searchScript); + var searchUrl = 'https://nominatim.openstreetmap.org/search?q=' + encodeURIComponent(searchString) + + '&format=json&limit=' + encodeURIComponent(searchLimit); + fetch(searchUrl) + .then(function(response) { return response.json(); }) + .then(parseResult) + .catch(function() { parseResult([]); }); } } +// Escape a value for use in HTML text and in double-quoted attributes. +function escapeSearchHtml(value) { + return String(value) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + function parseResult(response) { var searchResultDivs = ''; + if (!Array.isArray(response)) { + response = []; + } + if (response.length == searchLimit) { var lastResultIndex = searchLimit - 1; var nextSearchLimit = searchLimit + 10; @@ -67,10 +84,15 @@ function parseResult(response) { // Show the current 10 search results. for (var i=searchLimit-11; i' + - '
' + searchResult.class + ' / ' + searchResult.type + '
' + - '
' + searchResult.display_name + '
' + + '
' + + '
' + escapeSearchHtml(searchResult.class) + ' / ' + escapeSearchHtml(searchResult.type) + '
' + + '
' + escapeSearchHtml(searchResult.display_name) + '
' + '
'; } From 344232fb758e65cac6b778ce539b4f48cab638d3 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:36:49 +0100 Subject: [PATCH 07/30] fix: Hardens web container - document root is owned by root and not writable by www-data (the base image ships /var/www/html as www-data with mode 1777) - uses php.ini-production, so errors are not shown to clients - disables directory listings, expose_php and the detailed server signature --- web/Containerfile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/web/Containerfile b/web/Containerfile index e8b5998..c84f8a7 100644 --- a/web/Containerfile +++ b/web/Containerfile @@ -1,4 +1,17 @@ FROM docker.io/library/php:apache RUN docker-php-ext-install mysqli bcmath -COPY ./www/* /var/www/html \ No newline at end of file + +# Use PHP's production defaults (no error output to clients) and hide the PHP version. +RUN cp "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" \ + && printf 'expose_php = Off\n' > "$PHP_INI_DIR/conf.d/zz-hardening.ini" + +# Disable directory listings and reduce the server signature. +RUN sed -i 's/^\(\s*Options\) Indexes FollowSymLinks/\1 FollowSymLinks/' /etc/apache2/apache2.conf \ + && sed -i -e 's/^ServerTokens .*/ServerTokens Prod/' -e 's/^ServerSignature .*/ServerSignature Off/' /etc/apache2/conf-available/security.conf + +COPY ./www/* /var/www/html + +# The web server must not be able to create or modify files in the document root. +RUN chown -R root:root /var/www/html \ + && chmod -R u=rwX,go=rX /var/www/html From 189105075c718607722eef940b115717133d79ea Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:36:49 +0100 Subject: [PATCH 08/30] fix: Publishes images only from main Every pushed ref built and pushed images, with the tag derived from the ref name. A branch named "latest" or a git tag named "main" therefore overwrote the production image, and ref names were interpolated into shell scripts unquoted (script injection). - images are only pushed for pushes to refs/heads/main - other refs are built and tested with a ci- tag, never pushed - registry login happens only when publishing, after the compose test - main additionally pushes an immutable sha- tag for pinning - expression values are passed to scripts via env The registry credentials still need to be restricted to main on the Forgejo side, since a pushed branch can modify this workflow. --- .forgejo/workflows/images.yml | 91 +++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 32 deletions(-) diff --git a/.forgejo/workflows/images.yml b/.forgejo/workflows/images.yml index 6dfac7b..87f45e6 100644 --- a/.forgejo/workflows/images.yml +++ b/.forgejo/workflows/images.yml @@ -4,6 +4,16 @@ on: push: workflow_dispatch: +# Images are only published for pushes to the main branch. All other refs (branches, +# renovate branches, tags) are built and tested locally, but never pushed. +# +# Note: a workflow run for a pushed ref uses the workflow file from that ref, so this +# condition alone does not stop someone with push access from publishing images. The +# registry credentials must additionally be restricted to the protected main branch. + +env: + IMAGE_PREFIX: git.hamburg.ccc.de/ccchh/sunders + jobs: build: runs-on: docker @@ -14,44 +24,61 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Set image tags + id: vars + env: + COMMIT_SHA: ${{ github.sha }} + PUBLISH: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + run: | + if [ "$PUBLISH" = "true" ]; then + echo "tag=latest" >> "$GITHUB_OUTPUT" + else + # Never derive tags from ref names: they are attacker-controlled and may collide with `latest`. + echo "tag=ci-${COMMIT_SHA}" >> "$GITHUB_OUTPUT" + fi + echo "sha_tag=sha-${COMMIT_SHA}" >> "$GITHUB_OUTPUT" + + - name: Build web image + env: + TAG: ${{ steps.vars.outputs.tag }} + run: | + docker build -f ./Containerfile -t "$IMAGE_PREFIX/web:$TAG" . + working-directory: ./web + + - name: Build data_handler image + env: + TAG: ${{ steps.vars.outputs.tag }} + run: | + docker build -f ./Containerfile -t "$IMAGE_PREFIX/data_handler:$TAG" . + working-directory: ./data_handler + + - name: Update docker-compose.yml image tags + env: + TAG: ${{ steps.vars.outputs.tag }} + run: | + sed -i "s/:latest/:${TAG}/g" docker-compose.yml + + - name: Start Docker Compose services + run: | + docker compose up -d --wait + docker compose down + - name: Login to Container Registry + if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: docker/login-action@v3 with: registry: git.hamburg.ccc.de username: ${{ secrets.REGISTRY_USERNAME }} password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Set image tags - id: vars - run: | - if [ "${{ github.ref_name }}" = "main" ] && [ "${{ github.event_name }}" = "push" ]; then - echo "tag=latest" >> $GITHUB_OUTPUT - else - # renovate creates sub branches with `/`; these break the tagging in the build process - echo tag=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') >> $GITHUB_OUTPUT - fi - - - name: Build web image - run: | - docker build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} . - working-directory: ./web - - - name: Build data_handler image - run: | - docker build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} . - working-directory: ./data_handler - name: Push images to Container Registry + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + TAG: ${{ steps.vars.outputs.tag }} + SHA_TAG: ${{ steps.vars.outputs.sha_tag }} run: | - docker push git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} - docker push git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} - if: github.event_name == 'push' - - - name: Update docker-compose.yml image tags - run: | - sed -i "s/:latest/:${{ steps.vars.outputs.tag }}/g" docker-compose.yml - - - name: Start Docker Compose services - run: | - docker compose up -d --wait - docker compose down \ No newline at end of file + for image in web data_handler; do + docker tag "$IMAGE_PREFIX/$image:$TAG" "$IMAGE_PREFIX/$image:$SHA_TAG" + docker push "$IMAGE_PREFIX/$image:$TAG" + docker push "$IMAGE_PREFIX/$image:$SHA_TAG" + done From 3b2828033342266459bd98cd9496d45df8e2a605 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:36:49 +0100 Subject: [PATCH 09/30] fix: Escapes generated SQL and validates sequence number create_camera_update_statements.php wrote OSM tag keys and values unescaped into /migrations/*.sql. Values are now escaped and node ids cast to int. The sequence number argument must be numeric and is bound as a parameter instead of being concatenated into SQL and the output path. --- .../utils/create_camera_update_statements.php | 40 +++++++++++++------ data_handler/utils/update_camera.php | 12 +++++- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/data_handler/utils/create_camera_update_statements.php b/data_handler/utils/create_camera_update_statements.php index cf18dfc..a2a072c 100755 --- a/data_handler/utils/create_camera_update_statements.php +++ b/data_handler/utils/create_camera_update_statements.php @@ -2,6 +2,18 @@ include "config.php"; +$seqNum = $_SERVER['argv'][1] ?? ''; +if (!ctype_digit($seqNum)) { + echo "Invalid sequence number\n"; + exit(1); +} + +// Quote a value for the generated SQL file. Values come from OSM data and are untrusted. +function sqlString($value) { + global $mysqli; + return "'" . $mysqli->real_escape_string((string) $value) . "'"; +} + $elementTypes = array(); $count = 0; $countDelete = 0; @@ -91,7 +103,7 @@ function endElement ($parser, $name) { $mode = ''; } else if ($name == 'node') { if ($mode == 'delete') { - $id = $curNodeAttrs['id']; + $id = (int) $curNodeAttrs['id']; if (! $deleteTagStmt->execute()) { echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n"; @@ -119,7 +131,7 @@ function endElement ($parser, $name) { - $id = $curNodeAttrs['id']; + $id = (int) $curNodeAttrs['id']; if ($mode == 'modify') { if (! $deleteTagStmt->execute()) { @@ -140,36 +152,36 @@ function endElement ($parser, $name) { $k = 'lat'; $v = $curNodeAttrs['lat']; - $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, " . sqlString($k) . ", " . sqlString($v) . ");\n"; $k = 'lon'; $v = $curNodeAttrs['lon']; - $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, " . sqlString($k) . ", " . sqlString($v) . ");\n"; $k = 'userid'; $v = $curNodeAttrs['user']; - $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, " . sqlString($k) . ", " . sqlString($v) . ");\n"; $k = 'version'; $v = $curNodeAttrs['version']; - $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, " . sqlString($k) . ", " . sqlString($v) . ");\n"; $k = 'timestamp'; $v = $curNodeAttrs['timestamp']; - $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, " . sqlString($k) . ", " . sqlString($v) . ");\n"; foreach($curNodeTags as $k => $v) { - $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, " . sqlString($k) . ", " . sqlString($v) . ");\n"; } } else if ($mode == 'modify') { // delete former surveillance nodes that were modified to non-surveillance nodes - $id = $curNodeAttrs['id']; + $id = (int) $curNodeAttrs['id']; if (! $deleteTagStmt->execute()) { echo "***** Error : Deleting tags $id for non-surveillance node : ". $deleteTagStmt->error . "\n"; @@ -208,11 +220,13 @@ while ($data = fread($fp, 4096)) { } } -$sql="UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';"; -$mysqli->query($sql); -$sql_statement = $sql_statement . "UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';\n"; +$syncStmt = $mysqli->prepare("UPDATE sync_state SET v=? WHERE k='sequenceNumber'"); +$syncStmt->bind_param('s', $seqNum); +$syncStmt->execute(); +$syncStmt->close(); +$sql_statement = $sql_statement . "UPDATE sync_state SET v='" . $seqNum ."' WHERE k='sequenceNumber';\n"; $sql_statement = $sql_statement . "COMMIT;"; -$sql_file = "/migrations/".$_SERVER['argv'][1].".sql"; +$sql_file = "/migrations/".$seqNum.".sql"; file_put_contents($sql_file, $sql_statement); $mysqli->close(); diff --git a/data_handler/utils/update_camera.php b/data_handler/utils/update_camera.php index dc88e48..539e0ca 100755 --- a/data_handler/utils/update_camera.php +++ b/data_handler/utils/update_camera.php @@ -2,6 +2,12 @@ include "config.php"; +$seqNum = $_SERVER['argv'][1] ?? ''; +if (!ctype_digit($seqNum)) { + echo "Invalid sequence number\n"; + exit(1); +} + $elementTypes = array(); $count = 0; $countDelete = 0; @@ -281,8 +287,10 @@ while ($data = fread($fp, 4096)) { } } -$sql="UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';"; -$mysqli->query($sql); +$syncStmt = $mysqli->prepare("UPDATE sync_state SET v=? WHERE k='sequenceNumber'"); +$syncStmt->bind_param('s', $seqNum); +$syncStmt->execute(); +$syncStmt->close(); $mysqli->commit(); printDebug(); From cdf90a04d32c60ee3dc7805a1916f0ccb87ea37d Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:36:49 +0100 Subject: [PATCH 10/30] fix: Corrects country code check in statistics filter Operator precedence made the check always true; it now accepts two capital letters or '--' as intended. --- web/www/sunders/stats/index.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/www/sunders/stats/index.php b/web/www/sunders/stats/index.php index 7a0dac4..56ef660 100644 --- a/web/www/sunders/stats/index.php +++ b/web/www/sunders/stats/index.php @@ -84,8 +84,8 @@ } else { $i18n = $i18nCountries; $i18nDefault = $i18nCountriesDefault; - // Check if $key consists of 2 capital letters. - $key = preg_match('/\b[A-Z]{2}\b/', $valsArray[$i]) === 1 || '--' ? $valsArray[$i] : 'dontfindme'; + // Check if $key consists of 2 capital letters or is '--' (unknown country). + $key = (preg_match('/^[A-Z]{2}$/', $valsArray[$i]) === 1 || $valsArray[$i] === '--') ? $valsArray[$i] : 'dontfindme'; } if (!isValidValue($i18n, $i18nDefault, $key)) { // ignore 'vals' and 'cols' From da84719d3cca449b89f2198d85ec65d6ca1584f7 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:47:13 +0100 Subject: [PATCH 11/30] fix: Removes trailing comma from links.json Removing the last German entry left a trailing comma, which makes json_decode() fail and hides the whole links section. --- web/www/sunders/json/links.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/www/sunders/json/links.json b/web/www/sunders/json/links.json index d7cd3cc..3f8aeda 100644 --- a/web/www/sunders/json/links.json +++ b/web/www/sunders/json/links.json @@ -42,7 +42,7 @@ { "sourceText": "EPICENTER.WORKS", "href": "https://überwachungspaket.at/", "linkText": "Stoppt das Überwachungspaket!" }, { "sourceText": "Frag den Staat", "href": "https://fragdenstaat.de/anfrage/informationen-zum-projekt-sicherheitsbahnhof-berlin-sudkreuz/", "linkText": "Informationen zum „Projekt Sicherheitsbahnhof“ Berlin Südkreuz" }, { "sourceText": "Frag den Staat", "href": "https://fragdenstaat.de/anfrage/datenschutzkonzept-fur-videouberwachung-am-bahnhof-berlin-sudkreuz/", "linkText": "Datenschutzkonzept für Videoüberwachung am Bahnhof Berlin Südkreuz" }, - { "sourceText": "Reclaim Your Face", "href": "https://reclaimyourface.eu/de/", "linkText": "Reclaim Your Face DE" }, + { "sourceText": "Reclaim Your Face", "href": "https://reclaimyourface.eu/de/", "linkText": "Reclaim Your Face DE" } ] } ] } From 6ec1b9fc5a0c0fc3fa4825056abdea7245d60b7e Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:48:03 +0100 Subject: [PATCH 12/30] fix: Builds images with kaniko There are no Windows runners; build the images with kaniko in a container on the `docker` runner, as on the link-updates branch, while keeping the publishing restrictions: - only pushes to refs/heads/main publish, as `latest` and `sha-` - other refs build with --no-push under a ci- tag - registry credentials are only passed to the build when publishing The docker compose smoke test is dropped, as there is no Docker daemon inside the kaniko container. --- .forgejo/workflows/images.yml | 85 ++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/.forgejo/workflows/images.yml b/.forgejo/workflows/images.yml index 462bab2..30a037b 100644 --- a/.forgejo/workflows/images.yml +++ b/.forgejo/workflows/images.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: # Images are only published for pushes to the main branch. All other refs (branches, -# renovate branches, tags) are built and tested locally, but never pushed. +# renovate branches, tags) are built, but never pushed, and do not get registry credentials. # # Note: a workflow run for a pushed ref uses the workflow file from that ref, so this # condition alone does not stop someone with push access from publishing images. The @@ -16,23 +16,27 @@ env: jobs: build: - runs-on: windows-2022 + runs-on: docker + container: + image: ghcr.io/osscontainertools/kaniko:alpine steps: - - name: Checkout repository - uses: actions/checkout@v7 + - name: Install required system packages + run: apk add --no-cache nodejs git - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + - name: Checkout repository + uses: actions/checkout@v4 - name: Set image tags id: vars env: - COMMIT_SHA: ${{ github.sha }} - PUBLISH: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + COMMIT_SHA: ${{ forgejo.sha }} + PUBLISH: ${{ forgejo.event_name == 'push' && forgejo.ref == 'refs/heads/main' }} run: | if [ "$PUBLISH" = "true" ]; then + echo "publish=true" >> "$GITHUB_OUTPUT" echo "tag=latest" >> "$GITHUB_OUTPUT" else + echo "publish=false" >> "$GITHUB_OUTPUT" # Never derive tags from ref names: they are attacker-controlled and may collide with `latest`. echo "tag=ci-${COMMIT_SHA}" >> "$GITHUB_OUTPUT" fi @@ -40,45 +44,42 @@ jobs: - name: Build web image env: + IMAGE: web + PUBLISH: ${{ steps.vars.outputs.publish }} TAG: ${{ steps.vars.outputs.tag }} + SHA_TAG: ${{ steps.vars.outputs.sha_tag }} + KANIKO_GIT_HAMBURG_CCC_DE_USER: ${{ steps.vars.outputs.publish == 'true' && secrets.REGISTRY_USERNAME || '' }} + KANIKO_GIT_HAMBURG_CCC_DE_PASSWORD: ${{ steps.vars.outputs.publish == 'true' && secrets.REGISTRY_TOKEN || '' }} run: | - docker buildx build -f ./Containerfile -t "$IMAGE_PREFIX/web:$TAG" --load . - working-directory: ./web + if [ "$PUBLISH" = "true" ]; then + set -- --destination="$IMAGE_PREFIX/$IMAGE:$TAG" --destination="$IMAGE_PREFIX/$IMAGE:$SHA_TAG" + else + set -- --destination="$IMAGE_PREFIX/$IMAGE:$TAG" --no-push + fi + /kaniko/executor \ + --dockerfile="$GITHUB_WORKSPACE/$IMAGE/Containerfile" \ + --context="dir://$GITHUB_WORKSPACE/$IMAGE" \ + --no-push-cache \ + --credential-helpers=env \ + "$@" - name: Build data_handler image env: - TAG: ${{ steps.vars.outputs.tag }} - run: | - docker buildx build -f ./Containerfile -t "$IMAGE_PREFIX/data_handler:$TAG" --load . - working-directory: ./data_handler - - - name: Update docker-compose.yml image tags - env: - TAG: ${{ steps.vars.outputs.tag }} - run: | - sed -i "s/:latest/:${TAG}/g" docker-compose.yml - - - name: Start Docker Compose services - run: | - docker compose up -d --wait - docker compose down - - - name: Login to Container Registry - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - uses: docker/login-action@v4 - with: - registry: git.hamburg.ccc.de - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_TOKEN }} - - - name: Push images to Container Registry - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - env: + IMAGE: data_handler + PUBLISH: ${{ steps.vars.outputs.publish }} TAG: ${{ steps.vars.outputs.tag }} SHA_TAG: ${{ steps.vars.outputs.sha_tag }} + KANIKO_GIT_HAMBURG_CCC_DE_USER: ${{ steps.vars.outputs.publish == 'true' && secrets.REGISTRY_USERNAME || '' }} + KANIKO_GIT_HAMBURG_CCC_DE_PASSWORD: ${{ steps.vars.outputs.publish == 'true' && secrets.REGISTRY_TOKEN || '' }} run: | - for image in web data_handler; do - docker tag "$IMAGE_PREFIX/$image:$TAG" "$IMAGE_PREFIX/$image:$SHA_TAG" - docker push "$IMAGE_PREFIX/$image:$TAG" - docker push "$IMAGE_PREFIX/$image:$SHA_TAG" - done + if [ "$PUBLISH" = "true" ]; then + set -- --destination="$IMAGE_PREFIX/$IMAGE:$TAG" --destination="$IMAGE_PREFIX/$IMAGE:$SHA_TAG" + else + set -- --destination="$IMAGE_PREFIX/$IMAGE:$TAG" --no-push + fi + /kaniko/executor \ + --dockerfile="$GITHUB_WORKSPACE/$IMAGE/Containerfile" \ + --context="dir://$GITHUB_WORKSPACE/$IMAGE" \ + --no-push-cache \ + --credential-helpers=env \ + "$@" From 46b7509c37d92b50f5710f84ddccc2862daacf8e Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:57:18 +0100 Subject: [PATCH 13/30] chore: Reverts workflow changes The image build workflow is handled separately in the kaniko-build branch; this pull request no longer changes it. --- .forgejo/workflows/images.yml | 98 +++++++++++++---------------------- 1 file changed, 35 insertions(+), 63 deletions(-) diff --git a/.forgejo/workflows/images.yml b/.forgejo/workflows/images.yml index 30a037b..0306fee 100644 --- a/.forgejo/workflows/images.yml +++ b/.forgejo/workflows/images.yml @@ -4,82 +4,54 @@ on: push: workflow_dispatch: -# Images are only published for pushes to the main branch. All other refs (branches, -# renovate branches, tags) are built, but never pushed, and do not get registry credentials. -# -# Note: a workflow run for a pushed ref uses the workflow file from that ref, so this -# condition alone does not stop someone with push access from publishing images. The -# registry credentials must additionally be restricted to the protected main branch. - -env: - IMAGE_PREFIX: git.hamburg.ccc.de/ccchh/sunders - jobs: build: - runs-on: docker - container: - image: ghcr.io/osscontainertools/kaniko:alpine + runs-on: windows-2022 steps: - - name: Install required system packages - run: apk add --no-cache nodejs git - - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to Container Registry + uses: docker/login-action@v4 + with: + registry: git.hamburg.ccc.de + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Set image tags id: vars - env: - COMMIT_SHA: ${{ forgejo.sha }} - PUBLISH: ${{ forgejo.event_name == 'push' && forgejo.ref == 'refs/heads/main' }} run: | - if [ "$PUBLISH" = "true" ]; then - echo "publish=true" >> "$GITHUB_OUTPUT" - echo "tag=latest" >> "$GITHUB_OUTPUT" + if [ "${{ github.ref_name }}" = "main" ] && [ "${{ github.event_name }}" = "push" ]; then + echo "tag=latest" >> $GITHUB_OUTPUT else - echo "publish=false" >> "$GITHUB_OUTPUT" - # Never derive tags from ref names: they are attacker-controlled and may collide with `latest`. - echo "tag=ci-${COMMIT_SHA}" >> "$GITHUB_OUTPUT" + # renovate creates sub branches with `/`; these break the tagging in the build process + echo tag=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') >> $GITHUB_OUTPUT fi - echo "sha_tag=sha-${COMMIT_SHA}" >> "$GITHUB_OUTPUT" - name: Build web image - env: - IMAGE: web - PUBLISH: ${{ steps.vars.outputs.publish }} - TAG: ${{ steps.vars.outputs.tag }} - SHA_TAG: ${{ steps.vars.outputs.sha_tag }} - KANIKO_GIT_HAMBURG_CCC_DE_USER: ${{ steps.vars.outputs.publish == 'true' && secrets.REGISTRY_USERNAME || '' }} - KANIKO_GIT_HAMBURG_CCC_DE_PASSWORD: ${{ steps.vars.outputs.publish == 'true' && secrets.REGISTRY_TOKEN || '' }} run: | - if [ "$PUBLISH" = "true" ]; then - set -- --destination="$IMAGE_PREFIX/$IMAGE:$TAG" --destination="$IMAGE_PREFIX/$IMAGE:$SHA_TAG" - else - set -- --destination="$IMAGE_PREFIX/$IMAGE:$TAG" --no-push - fi - /kaniko/executor \ - --dockerfile="$GITHUB_WORKSPACE/$IMAGE/Containerfile" \ - --context="dir://$GITHUB_WORKSPACE/$IMAGE" \ - --no-push-cache \ - --credential-helpers=env \ - "$@" + docker buildx build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} --load . + working-directory: ./web - name: Build data_handler image - env: - IMAGE: data_handler - PUBLISH: ${{ steps.vars.outputs.publish }} - TAG: ${{ steps.vars.outputs.tag }} - SHA_TAG: ${{ steps.vars.outputs.sha_tag }} - KANIKO_GIT_HAMBURG_CCC_DE_USER: ${{ steps.vars.outputs.publish == 'true' && secrets.REGISTRY_USERNAME || '' }} - KANIKO_GIT_HAMBURG_CCC_DE_PASSWORD: ${{ steps.vars.outputs.publish == 'true' && secrets.REGISTRY_TOKEN || '' }} run: | - if [ "$PUBLISH" = "true" ]; then - set -- --destination="$IMAGE_PREFIX/$IMAGE:$TAG" --destination="$IMAGE_PREFIX/$IMAGE:$SHA_TAG" - else - set -- --destination="$IMAGE_PREFIX/$IMAGE:$TAG" --no-push - fi - /kaniko/executor \ - --dockerfile="$GITHUB_WORKSPACE/$IMAGE/Containerfile" \ - --context="dir://$GITHUB_WORKSPACE/$IMAGE" \ - --no-push-cache \ - --credential-helpers=env \ - "$@" + docker buildx build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} --load . + working-directory: ./data_handler + + - name: Push images to Container Registry + run: | + docker push git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} + docker push git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} + if: github.event_name == 'push' + + - name: Update docker-compose.yml image tags + run: | + sed -i "s/:latest/:${{ steps.vars.outputs.tag }}/g" docker-compose.yml + + - name: Start Docker Compose services + run: | + docker compose up -d --wait + docker compose down \ No newline at end of file From d724be54bff95c223a2d251c891a78fc62ac10a6 Mon Sep 17 00:00:00 2001 From: Stefan Bethke Date: Sun, 15 Nov 2026 11:57:20 +0100 Subject: [PATCH 14/30] chore: Reverts workflow changes The image build workflow is handled separately in the kaniko-build branch; this pull request no longer changes it. --- .forgejo/workflows/images.yml | 62 +++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/.forgejo/workflows/images.yml b/.forgejo/workflows/images.yml index cb999c5..0306fee 100644 --- a/.forgejo/workflows/images.yml +++ b/.forgejo/workflows/images.yml @@ -6,48 +6,52 @@ on: jobs: build: - runs-on: docker - container: - image: ghcr.io/osscontainertools/kaniko:alpine + runs-on: windows-2022 steps: - - name: Install required system packages - run: apk add --no-cache nodejs git - - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to Container Registry + uses: docker/login-action@v4 + with: + registry: git.hamburg.ccc.de + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} - name: Set image tags id: vars run: | - if [ "${{ forgejo.ref_name }}" = "main" ] && [ "${{ forgejo.event_name }}" = "push" ]; then + if [ "${{ github.ref_name }}" = "main" ] && [ "${{ github.event_name }}" = "push" ]; then echo "tag=latest" >> $GITHUB_OUTPUT else # renovate creates sub branches with `/`; these break the tagging in the build process - echo tag=$(echo "${{ forgejo.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') >> $GITHUB_OUTPUT + echo tag=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') >> $GITHUB_OUTPUT fi - name: Build web image - env: - KANIKO_NO_PUSH: ${{ forgejo.event_name != 'push' }} - KANIKO_GIT_HAMBURG_CCC_DE_USER: ${{ secrets.REGISTRY_USERNAME }} - KANIKO_GIT_HAMBURG_CCC_DE_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} run: | - /kaniko/executor \ - --dockerfile="${{ forgejo.workspace }}/web/Containerfile" \ - --context="dir://${{ forgejo.workspace }}/web" \ - --destination=git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} \ - --no-push-cache \ - --credential-helpers=env + docker buildx build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} --load . + working-directory: ./web - name: Build data_handler image - env: - KANIKO_NO_PUSH: ${{ forgejo.event_name != 'push' }} - KANIKO_GIT_HAMBURG_CCC_DE_USER: ${{ secrets.REGISTRY_USERNAME }} - KANIKO_GIT_HAMBURG_CCC_DE_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} run: | - /kaniko/executor \ - --dockerfile="${{ forgejo.workspace }}/data_handler/Containerfile" \ - --context="dir://${{ forgejo.workspace }}/data_handler" \ - --destination=git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} \ - --no-push-cache \ - --credential-helpers=env \ No newline at end of file + docker buildx build -f ./Containerfile -t git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} --load . + working-directory: ./data_handler + + - name: Push images to Container Registry + run: | + docker push git.hamburg.ccc.de/ccchh/sunders/web:${{ steps.vars.outputs.tag }} + docker push git.hamburg.ccc.de/ccchh/sunders/data_handler:${{ steps.vars.outputs.tag }} + if: github.event_name == 'push' + + - name: Update docker-compose.yml image tags + run: | + sed -i "s/:latest/:${{ steps.vars.outputs.tag }}/g" docker-compose.yml + + - name: Start Docker Compose services + run: | + docker compose up -d --wait + docker compose down \ No newline at end of file From d794ee032c4c01f20660775d0e6833c4ed3d8fae Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 25 Oct 2025 19:49:34 +0200 Subject: [PATCH 15/30] feat: Adds healthcheck to web server --- docker-compose.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 8954b73..77cf550 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,14 @@ services: depends_on: data_handler: condition: service_started + # all the way zoomed out, with hamburg at the center, at least one camera should be returned + healthcheck: + test: > + ["CMD", "curl -s 'http://localhost/camera.php?bbox=-92.52991540527346,30.683278176916133,131.7689595947266,72.87186315234298&zoom=4&width=2552&height=867' | grep '^\[.+\]$'"] + interval: 10s + timeout: 5s + start_period: 30s + retries: 5 data_handler: image: git.hamburg.ccc.de/ccchh/sunders/data_handler:latest From 7332dd8d3c71c9f9c28745e4ad91546a1af7a529 Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 25 Oct 2025 21:20:09 +0200 Subject: [PATCH 16/30] refactor: Increases verbosity of data_handler --- data_handler/Containerfile | 2 +- data_handler/utils/entrypoint.sh | 2 +- data_handler/utils/update_camera.sh | 4 ++-- home/sunders/update_cameras/update_camera.sh | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/data_handler/Containerfile b/data_handler/Containerfile index 7215bf9..4c08fa6 100644 --- a/data_handler/Containerfile +++ b/data_handler/Containerfile @@ -5,4 +5,4 @@ COPY ./data_init/*.sql /opt/init/init.sql ENTRYPOINT ["/opt/entrypoint.sh"] -CMD ["/usr/sbin/dcrond", "-f" ] \ No newline at end of file +CMD ["/bin/sh", "-c", "/usr/sbin/dcrond -f -d 5 > /var/log/dcrond.log 2>&1"] \ No newline at end of file diff --git a/data_handler/utils/entrypoint.sh b/data_handler/utils/entrypoint.sh index c4dc346..2109d85 100755 --- a/data_handler/utils/entrypoint.sh +++ b/data_handler/utils/entrypoint.sh @@ -8,7 +8,7 @@ set -e cat<> /var/log/update_camera_$(date +\%Y\%m\%d\%H\%M\%S).log 2>&1 EOF #################################################################################################### diff --git a/data_handler/utils/update_camera.sh b/data_handler/utils/update_camera.sh index 25d23c8..4107d9f 100644 --- a/data_handler/utils/update_camera.sh +++ b/data_handler/utils/update_camera.sh @@ -63,7 +63,7 @@ do rm "change_file.osc.gz" fi - echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName" + echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum of $newSeqNum" > "$logFileName" targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'` targetDirName="$REPLICATE_URL/$targetDirName" @@ -87,7 +87,7 @@ do targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'` curl -L "$targetDirName" -o lastState.txt 2>&1 | tee -a $logFileName - echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName" + echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum of $newSeqNum" | tee -a "$logFileName" done rm "/var/lock/update_camera" diff --git a/home/sunders/update_cameras/update_camera.sh b/home/sunders/update_cameras/update_camera.sh index b605ee4..b6d3338 100644 --- a/home/sunders/update_cameras/update_camera.sh +++ b/home/sunders/update_cameras/update_camera.sh @@ -60,7 +60,7 @@ do rm "change_file.osc.gz" fi - echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName" + echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum of $newSeqNum" > "$logFileName" targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'` targetDirName="$REPLICATE_URL/$targetDirName" @@ -84,7 +84,7 @@ do targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'` wget -nv "$targetDirName" -O lastState.txt 2>&1 | tee -a $logFileName - echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName" + echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum of $newSeqNum" | tee -a "$logFileName" done rm "/var/lock/update_camera" From ddc73f7207cb7c39a888a0882da0ee41b3481edd Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 25 Oct 2025 21:20:27 +0200 Subject: [PATCH 17/30] chore: Immediately starts `update_camera` on container startup --- data_handler/utils/entrypoint.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/data_handler/utils/entrypoint.sh b/data_handler/utils/entrypoint.sh index 2109d85..9e726e8 100755 --- a/data_handler/utils/entrypoint.sh +++ b/data_handler/utils/entrypoint.sh @@ -20,7 +20,15 @@ php /opt/init_db.php echo "Prescripts done" #################################################################################################### -### Run cmd +### Run script once immediately +#################################################################################################### + +echo "Running update once initially..." +sh /opt/update_camera.sh >> /var/log/update_camera_$(date +\%Y\%m\%d\%H\%M\%S).log 2>&1 +echo "Initial update done" + +#################################################################################################### +### Run cron to schedule periodic task #################################################################################################### # see: https://github.com/dubiousjim/dcron/issues/13 From c53b37e8f0197de78a38508f28e7f4c9ef1fbef8 Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 25 Oct 2025 21:21:06 +0200 Subject: [PATCH 18/30] feat: Adds healthcheck that restarts `data_handler`, when any error is printed to log files --- docker-compose.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 77cf550..a74b7a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,6 +57,13 @@ services: db: condition: service_healthy restart: true + # if the latest `update_camera_*.log` contains "error", fail the healthcheck + healthcheck: + test: ["CMD", "sh", "-c", "ls /var/log | grep '^update_camera_' | xargs -I {} sh -c 'grep -q error /var/log/{} || exit 0; exit 1'"] + interval: 10s + timeout: 5s + start_period: 30s + retries: 5 volumes: mariadb: \ No newline at end of file From f1dd6f9b2f55a16d99e38efd5a8e86249018940e Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 25 Oct 2025 21:22:23 +0200 Subject: [PATCH 19/30] fix: Restarts any container unless stopped by user --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index a74b7a9..c957f4c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: db: image: mariadb:13.0.2 + restart: unless-stopped command: --max_allowed_packet=3250585600 environment: MYSQL_ROOT_PASSWORD: rootpassword # ${{secrets.MYSQL_ROOT_PASSWORD}} @@ -18,6 +19,7 @@ services: web: image: git.hamburg.ccc.de/ccchh/sunders/web:latest + restart: unless-stopped environment: MYSQL_HOST: db MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}} @@ -44,6 +46,7 @@ services: data_handler: image: git.hamburg.ccc.de/ccchh/sunders/data_handler:latest + restart: unless-stopped environment: MYSQL_HOST: db MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}} From 6f633a400c8173dbf1177ad79431dcaff12c7254 Mon Sep 17 00:00:00 2001 From: aso Date: Sat, 1 Nov 2025 12:20:36 +0100 Subject: [PATCH 20/30] first changes for sync of instances --- docker-compose-dev.yml | 72 ++++++++++++++++++++++++++++++++++ web/www/sunders/sync-state.php | 27 +++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 docker-compose-dev.yml create mode 100644 web/www/sunders/sync-state.php diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000..58c2af2 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,72 @@ +services: + db: + image: mariadb:12.0.2 + restart: unless-stopped + command: --max_allowed_packet=3250585600 + environment: + MYSQL_ROOT_PASSWORD: rootpassword # ${{secrets.MYSQL_ROOT_PASSWORD}} + MYSQL_DATABASE: camera # ${{secrets.MYSQL_DATABASE}} + MYSQL_USER: camera # ${{secrets.MYSQL_USER}} + MYSQL_PASSWORD: camerapassword # ${{secrets.MYSQL_PASSWORD}} + volumes: + - ./mariadb:/var/lib/mysql:Z + healthcheck: + test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-prootpassword"] + interval: 10s + timeout: 5s + start_period: 30s + retries: 5 + + web: + image: git.hamburg.ccc.de/ccchh/sunders/web:latest + restart: unless-stopped + environment: + MYSQL_HOST: db + MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}} + CAMERA_SELECT_USER: camera_select # ${{secrets.CAMERA_SELECT_USER}} + CAMERA_SELECT_USER_PASSWORD: camera_selectpassword # ${{secrets.CAMERA_SELECT_USER_PASSWORD}} + DEFAULT_ZOOM: 12 + DEFAULT_LAT: 0 + DEFAULT_LON: 0 + DEFAULT_LANGUAGE: en + IMPRESSUM_URL: https://hamburg.ccc.de/imprint/ + ports: + - "8081:80" + volumes: + - ./web/www/sunders:/var/www/html:Z + depends_on: + db: + condition: service_started + # all the way zoomed out, with hamburg at the center, at least one camera should be returned + healthcheck: + test: > + ["CMD", "curl -s 'http://localhost/camera.php?bbox=-92.52991540527346,30.683278176916133,131.7689595947266,72.87186315234298&zoom=4&width=2552&height=867' | grep '^\[.+\]$'"] + interval: 10s + timeout: 5s + retries: 5 + + data_handler: + image: git.hamburg.ccc.de/ccchh/sunders/data_handler:latest + restart: unless-stopped + environment: + MYSQL_HOST: db + MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}} + MYSQL_USER: root # ${{secrets.MYSQL_USER}} + MYSQL_PASSWORD: rootpassword # ${{secrets.MYSQL_ROOT_PASSWORD}} + CAMERA_USER: camera # ${{secrets.CAMERA_USER}} + CAMERA_USER_PASSWORD: camerapassword # ${{secrets.CAMERA_USER_PASSWORD}} + CAMERA_SELECT_USER: camera_select # ${{secrets.CAMERA_SELECT_USER}} + CAMERA_SELECT_USER_PASSWORD: camera_selectpassword # ${{secrets.CAMERA_SELECT_USER_PASSWORD}} + depends_on: + db: + condition: service_healthy + restart: true + # if the latest `update_camera_*.log` contains "error", fail the healthcheck + healthcheck: + test: ["CMD", "sh", "-c", "ls /var/log | grep '^update_camera_' | xargs -I {} sh -c 'grep -q error /var/log/{} || exit 0; exit 1'"] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + mariadb: \ No newline at end of file diff --git a/web/www/sunders/sync-state.php b/web/www/sunders/sync-state.php new file mode 100644 index 0000000..095e92c --- /dev/null +++ b/web/www/sunders/sync-state.php @@ -0,0 +1,27 @@ +connect_errno) { + header('Content-type: application/json'); + $result = '{"error":"error while connecting to db : ' . $mysqli->error . '"}'; + echo $result; + exit; + } + + $syncstate_querry = $mysqli->query("SELECT * FROM sync_state WHERE k = 'sequenceNumber'"); + + while($row = $syncstate_querry->fetch_assoc()) { + $syncstate = array('sequenceNumber' => $row["v"]); + } + + $result = json_encode($syncstate); + + $mysqli->close(); + + header('Content-type: application/json; Charset : utf-8'); + echo $result; +?> From c186886df71dd70ce0e835915709b96f35b326d1 Mon Sep 17 00:00:00 2001 From: aso Date: Sat, 1 Nov 2025 15:34:50 +0100 Subject: [PATCH 21/30] modified health check for web --- docker-compose-dev.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 58c2af2..28c7c46 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -39,11 +39,11 @@ services: condition: service_started # all the way zoomed out, with hamburg at the center, at least one camera should be returned healthcheck: - test: > - ["CMD", "curl -s 'http://localhost/camera.php?bbox=-92.52991540527346,30.683278176916133,131.7689595947266,72.87186315234298&zoom=4&width=2552&height=867' | grep '^\[.+\]$'"] + test: ["CMD", "curl", "-f", "http://localhost/sync-state.php"] interval: 10s timeout: 5s retries: 5 + start_period: 10s data_handler: image: git.hamburg.ccc.de/ccchh/sunders/data_handler:latest From 7727c56351bae73b7de6843526cc2597d991a0f1 Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 1 Nov 2025 20:55:49 +0100 Subject: [PATCH 22/30] feat: `web` healthcheck returns 503 status code if database is unavailable --- docker-compose.yml | 5 ++--- web/www/sunders/sync-state.php | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c957f4c..e97db01 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,12 +33,11 @@ services: ports: - "8081:80" depends_on: - data_handler: + db: condition: service_started # all the way zoomed out, with hamburg at the center, at least one camera should be returned healthcheck: - test: > - ["CMD", "curl -s 'http://localhost/camera.php?bbox=-92.52991540527346,30.683278176916133,131.7689595947266,72.87186315234298&zoom=4&width=2552&height=867' | grep '^\[.+\]$'"] + test: ["CMD", "curl", "-f", "http://localhost/sync-state.php"] interval: 10s timeout: 5s start_period: 30s diff --git a/web/www/sunders/sync-state.php b/web/www/sunders/sync-state.php index 095e92c..11ecef5 100644 --- a/web/www/sunders/sync-state.php +++ b/web/www/sunders/sync-state.php @@ -6,8 +6,9 @@ /* Connect to database */ $mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB); if($mysqli->connect_errno) { + http_response_code(503); header('Content-type: application/json'); - $result = '{"error":"error while connecting to db : ' . $mysqli->error . '"}'; + $result = '{"error":"Database unavailable"}'; echo $result; exit; } From 54e78f3f4ef6831c8d101dd7957f0d3bf46c221f Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 1 Nov 2025 21:23:38 +0100 Subject: [PATCH 23/30] docs: Adds instructions about `dev` override chore: Applies docker naming convention to files --- README.md | 6 ++-- docker-compose-dev.yml | 72 ------------------------------------------ docker-compose.dev.yml | 13 ++++++++ 3 files changed, 15 insertions(+), 76 deletions(-) delete mode 100644 docker-compose-dev.yml create mode 100644 docker-compose.dev.yml diff --git a/README.md b/README.md index b2e535c..e613464 100644 --- a/README.md +++ b/README.md @@ -35,12 +35,10 @@ To run Surveillance using docker, decide between development/testing or producti 1. Clone this repository -2. Run `docker compose up` and wait for services to start up +2. Run `docker compose -f docker-compose.yml -f docker-compose.dev.yml up` and wait for services to start up 3. Visit `http://localhost:8081/` in a browser; you should see an interactive OpenStreetMap -_tbd.: Populate camera data_ - ### Production 1. Close this repository @@ -51,7 +49,7 @@ _tbd.: Populate camera data_ 3. Visit `http://localhost:8081/` in a browser to open the web interface -_tbd.: Populate camera data_ +4. ## Surveillance nodes diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml deleted file mode 100644 index 28c7c46..0000000 --- a/docker-compose-dev.yml +++ /dev/null @@ -1,72 +0,0 @@ -services: - db: - image: mariadb:12.0.2 - restart: unless-stopped - command: --max_allowed_packet=3250585600 - environment: - MYSQL_ROOT_PASSWORD: rootpassword # ${{secrets.MYSQL_ROOT_PASSWORD}} - MYSQL_DATABASE: camera # ${{secrets.MYSQL_DATABASE}} - MYSQL_USER: camera # ${{secrets.MYSQL_USER}} - MYSQL_PASSWORD: camerapassword # ${{secrets.MYSQL_PASSWORD}} - volumes: - - ./mariadb:/var/lib/mysql:Z - healthcheck: - test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-prootpassword"] - interval: 10s - timeout: 5s - start_period: 30s - retries: 5 - - web: - image: git.hamburg.ccc.de/ccchh/sunders/web:latest - restart: unless-stopped - environment: - MYSQL_HOST: db - MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}} - CAMERA_SELECT_USER: camera_select # ${{secrets.CAMERA_SELECT_USER}} - CAMERA_SELECT_USER_PASSWORD: camera_selectpassword # ${{secrets.CAMERA_SELECT_USER_PASSWORD}} - DEFAULT_ZOOM: 12 - DEFAULT_LAT: 0 - DEFAULT_LON: 0 - DEFAULT_LANGUAGE: en - IMPRESSUM_URL: https://hamburg.ccc.de/imprint/ - ports: - - "8081:80" - volumes: - - ./web/www/sunders:/var/www/html:Z - depends_on: - db: - condition: service_started - # all the way zoomed out, with hamburg at the center, at least one camera should be returned - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost/sync-state.php"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s - - data_handler: - image: git.hamburg.ccc.de/ccchh/sunders/data_handler:latest - restart: unless-stopped - environment: - MYSQL_HOST: db - MYSQL_DB: camera # ${{secrets.MYSQL_DATABASE}} - MYSQL_USER: root # ${{secrets.MYSQL_USER}} - MYSQL_PASSWORD: rootpassword # ${{secrets.MYSQL_ROOT_PASSWORD}} - CAMERA_USER: camera # ${{secrets.CAMERA_USER}} - CAMERA_USER_PASSWORD: camerapassword # ${{secrets.CAMERA_USER_PASSWORD}} - CAMERA_SELECT_USER: camera_select # ${{secrets.CAMERA_SELECT_USER}} - CAMERA_SELECT_USER_PASSWORD: camera_selectpassword # ${{secrets.CAMERA_SELECT_USER_PASSWORD}} - depends_on: - db: - condition: service_healthy - restart: true - # if the latest `update_camera_*.log` contains "error", fail the healthcheck - healthcheck: - test: ["CMD", "sh", "-c", "ls /var/log | grep '^update_camera_' | xargs -I {} sh -c 'grep -q error /var/log/{} || exit 0; exit 1'"] - interval: 10s - timeout: 5s - retries: 5 - -volumes: - mariadb: \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..15542fa --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,13 @@ +services: + db: + volumes: + - ./mariadb:/var/lib/mysql:Z + + web: + volumes: + - ./web/www/sunders:/var/www/html:Z + + data_handler: + volumes: + - ./data_handler/utils:/opt:Z + - ./data_handler/data_init:/opt/init/init.sql:Z \ No newline at end of file From d9529cf6c088105d5534b1438ef7aeef5052bc5d Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 1 Nov 2025 21:27:47 +0100 Subject: [PATCH 24/30] fix: Enables PHP log output when running in `development` --- .gitignore | 5 ++++- docker-compose.dev.yml | 2 ++ web/www/sunders/add-lists.php | 1 - web/www/sunders/camera.php | 1 - web/www/sunders/config.php | 11 +++++++++++ 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 09f94a6..1febf73 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ lastState.txt lastStatisticsUpdate.txt log.* state.txt -migrations \ No newline at end of file +migrations + +# created when running development environment +mariadb/** \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 15542fa..61d0016 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -6,6 +6,8 @@ services: web: volumes: - ./web/www/sunders:/var/www/html:Z + environment: + APP_ENV: development data_handler: volumes: diff --git a/web/www/sunders/add-lists.php b/web/www/sunders/add-lists.php index da9f396..12fc0c0 100644 --- a/web/www/sunders/add-lists.php +++ b/web/www/sunders/add-lists.php @@ -1,5 +1,4 @@ \ No newline at end of file From ea5e30530476b8d91f3c4dca7739c4d459e565fb Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 1 Nov 2025 21:28:30 +0100 Subject: [PATCH 25/30] fix: Handles issue where error handler was skipped, if database DNS request fails --- web/www/sunders/sync-state.php | 40 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/web/www/sunders/sync-state.php b/web/www/sunders/sync-state.php index 11ecef5..3cb8ef0 100644 --- a/web/www/sunders/sync-state.php +++ b/web/www/sunders/sync-state.php @@ -2,27 +2,29 @@ error_reporting(0); include $pathToWebFolder.'config.php'; - - /* Connect to database */ - $mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB); - if($mysqli->connect_errno) { - http_response_code(503); - header('Content-type: application/json'); - $result = '{"error":"Database unavailable"}'; - echo $result; - exit; - } - - $syncstate_querry = $mysqli->query("SELECT * FROM sync_state WHERE k = 'sequenceNumber'"); - - while($row = $syncstate_querry->fetch_assoc()) { - $syncstate = array('sequenceNumber' => $row["v"]); + try { + $mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB); + if($mysqli->connect_errno) { + throw new Exception("Could not connect to database"); } - $result = json_encode($syncstate); + $syncstate_querry = $mysqli->query("SELECT * FROM sync_state WHERE k = 'sequenceNumber'"); - $mysqli->close(); + while($row = $syncstate_querry->fetch_assoc()) { + $syncstate = array('sequenceNumber' => $row["v"]); + } - header('Content-type: application/json; Charset : utf-8'); - echo $result; + $result = json_encode($syncstate); + + $mysqli->close(); + + header('Content-type: application/json; Charset : utf-8'); + echo $result; + mysqli_report(MYSQLI_REPORT_OFF); + } catch (Exception $e) { + http_response_code(503); + header('Content-type: application/json'); + $result = '{"error":"Database unavailable"}'; + echo $result; + } ?> From b6c05e526507b162890354509867aeb68dd0a05f Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 1 Nov 2025 21:28:59 +0100 Subject: [PATCH 26/30] docs: Removes incorrect comment --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index e97db01..3fae64e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,6 @@ services: depends_on: db: condition: service_started - # all the way zoomed out, with hamburg at the center, at least one camera should be returned healthcheck: test: ["CMD", "curl", "-f", "http://localhost/sync-state.php"] interval: 10s From f5ef9adc4db4534cde804892eac9347030b78ceb Mon Sep 17 00:00:00 2001 From: Vincent Mahnke Date: Sat, 1 Nov 2025 21:29:18 +0100 Subject: [PATCH 27/30] docs: Adds information about changes when running in a development environment --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e613464..a46ecb0 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,17 @@ To run Surveillance using docker, decide between development/testing or producti 3. Visit `http://localhost:8081/` in a browser; you should see an interactive OpenStreetMap +4. You can see most cameras on the map immediately; the `data_handler` background process continuously applies updates from OpenStreetMap which become visible after refrshing the page + +5. Appending `-f docker-compose.dev.yml` applies these changes: + + - PHP won't supress any errors; to disable this, remove `return;` in `./web/www/sunders/config.php` + + - Making changes in the `web` will immediately apply them to the running containers; try modifying `web/www/sunders/index.php` + + - Making changes in the `data_handler` will immediately apply them to the running containers (try modifying `data_handler/utils/update_camera.sh`) + + - The host has access to database files at `./mariadb` ### Production 1. Close this repository @@ -49,7 +60,7 @@ To run Surveillance using docker, decide between development/testing or producti 3. Visit `http://localhost:8081/` in a browser to open the web interface -4. +4. You can see most cameras on the map immediately; the `data_handler` background process continuously applies updates from OpenStreetMap which become visible after refrshing the page ## Surveillance nodes From 3013bf42b1c086439ad000a7f997a80034e53c56 Mon Sep 17 00:00:00 2001 From: ViMaSter Date: Sat, 1 Nov 2025 21:49:46 +0100 Subject: [PATCH 28/30] fix: Adds missing documentation hint for secret replacement --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3fae64e..99ed328 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: volumes: - mariadb:/var/lib/mysql healthcheck: - test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-prootpassword"] + test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-uroot", "-prootpassword"] # ${{secrets.MYSQL_ROOT_PASSWORD}} interval: 10s timeout: 5s start_period: 30s From 13f02116248d5f0ea9e9fa57bb0d4cee477cf1e9 Mon Sep 17 00:00:00 2001 From: ViMaSter Date: Sun, 2 Nov 2025 12:47:33 +0100 Subject: [PATCH 29/30] revert: Uses previous port mapping --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 99ed328..290227d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: DEFAULT_LANGUAGE: en IMPRESSUM_URL: https://hamburg.ccc.de/imprint/ ports: - - "8081:80" + - "8080:80" depends_on: db: condition: service_started From aa6ef7f3626cb78fb8f3fab710b7ae06a10b500c Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:24:47 +0100 Subject: [PATCH 30/30] fix: Pin PHP to php2shell-patched release Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- web/Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/Containerfile b/web/Containerfile index c84f8a7..984e64c 100644 --- a/web/Containerfile +++ b/web/Containerfile @@ -1,4 +1,4 @@ -FROM docker.io/library/php:apache +FROM docker.io/library/php:8.5.12-apache RUN docker-php-ext-install mysqli bcmath