CCCHH/sunders
CCCHH/sunders
3
0
Fork 1
forked from kamba4/sunders
Code Issues 5 Pull requests 3 Packages 1 Activity Actions

improved performance
All checks were successful
Build (and tag) Images / build (push) Successful in 38s
Details

Browse source
This commit is contained in:
aso 2025-10-14 21:37:29 +02:00
commit 8612798373
3 changed files with 58 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

99
data_handler/utils/create_camera_update_statements.php
View file

@ -57,50 +57,6 @@ if (! ($insertTagStmt = $mysqli->prepare("INSERT INTO tag (id, k, v) VALUES (?,
}
$insertTagStmt->bind_param('dss', $id, $k, $v);
if (USE_STATISTICS) {
if (! ($insertStatsStmt = $mysqli->prepare("INSERT INTO statistics (id, ts, version) VALUES (?, ?, ?)"))) {
echo "Error while preparing insert statistics statement : " . $mysqli->error ;
exit(1);
}
$insertStatsStmt->bind_param('dsi', $id, $ts, $version);
}
function printDebug() {
global $elementTypes, $count, $countDelete, $countModifyDelete, $countModify, $countCreate;
echo "== $count ==============================\n";
foreach($elementTypes as $k => $v) {
echo "$k : $v\n";
}
echo "++++++ Surveillance nodes : ++++++\n";
echo "$countDelete deletions\n";
echo "$countModifyDelete modifications --> deletions\n";
echo "$countModify modifications\n";
echo "$countCreate creations\n";
}
function printDebugCurNode() {
global $curNodeAttrs, $curNodeTags, $mode;
echo "=============\n";
echo "$mode : " . $curNodeAttrs['id'] ." (". $curNodeAttrs['lat'] . " x " . $curNodeAttrs['lon'] . ") : " . $curNodeAttrs['user'] . "\n";
if (! empty($curNodeAttrs)) {
echo " => Attributes :\n";
foreach($curNodeAttrs as $k => $v) {
echo " $k : $v\n";
}
} else {
echo " => No attributes\n";
}
if (! empty($curNodeTags)) {
echo " => Tags :\n";
foreach($curNodeTags as $k => $v) {
echo " $k : $v\n";
}
} else {
echo " => No tags\n";
}
}
function startElement ($parser, $name, $attrs) {
global $elementTypes, $count, $mode, $curNodeAttrs, $curNodeTags;
@ -137,12 +93,6 @@ function endElement ($parser, $name) {
if ($mode == 'delete') {
$id = $curNodeAttrs['id'];
if (USE_STATISTICS) {
if (! $deleteStatsStmt->execute()) {
echo "***** Error : Deleting statistics $id : ". $deleteStatsStmt->error . "\n";
}
}
if (! $deleteTagStmt->execute()) {
echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n";
}
@ -151,6 +101,13 @@ function endElement ($parser, $name) {
echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n";
}
if ($deleteStmt->affected_rows > 0) {
if (! $deleteTagStmt->execute()) {
echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n";
}
if (! $deleteStmt->execute()) {
echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n";
}
$sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
$sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
}
@ -165,19 +122,14 @@ function endElement ($parser, $name) {
$id = $curNodeAttrs['id'];
if ($mode == 'modify') {
if (USE_STATISTICS) {
if (! $deleteStatsStmt->execute()) {
echo "***** Error : Deleting statistics $id for modification : ". $deleteStatsStmt->error . "\n";
}
}
if (! $deleteTagStmt->execute()) {
echo "***** Error : Deleting tags $id for modification : ". $deleteTagStmt->error . "\n";
}
if (! $deleteStmt->execute()) {
echo "***** Error : Deleting $id for modification : ". $deleteStmt->error . "\n";
}
$sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
$sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
} else {
}
@ -198,57 +150,30 @@ function endElement ($parser, $name) {
$k = 'lon';
$v = $curNodeAttrs['lon'];
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
if (! $insertTagStmt->execute()) {
echo "***** Error : inserting longitude $v for $id : ". $insertTagStmt->error . "\n";
}
$k = 'userid';
$v = $curNodeAttrs['user'];
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
if (! $insertTagStmt->execute()) {
echo "***** Error : inserting user $v for $id : ". $insertTagStmt->error . "\n";
}
$k = 'version';
$v = $curNodeAttrs['version'];
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
if (! $insertTagStmt->execute()) {
echo "***** Error : inserting version $v for $id : ". $insertTagStmt->error . "\n";
}
$k = 'timestamp';
$v = $curNodeAttrs['timestamp'];
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
if (! $insertTagStmt->execute()) {
echo "***** Error : inserting timestamp $v for $id : ". $insertTagStmt->error . "\n";
}
foreach($curNodeTags as $k => $v) {
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
if (! $insertTagStmt->execute()) {
echo "***** Error : inserting tag $k => $v for $id : ". $insertTagStmt->error . "\n";
}
}
if (USE_STATISTICS) {
$ts = $curNodeAttrs['timestamp'];
$version = $curNodeAttrs['version'];
if (! $insertStatsStmt->execute()) {
echo "***** Error : inserting ts $ts, version $version for $id : ". $insertStatsStmt->error . "\n";
}
}
$mysqli->commit();
} else if ($mode == 'modify') {
// delete former surveillance nodes that were modified to non-surveillance nodes
$id = $curNodeAttrs['id'];
if (USE_STATISTICS) {
if (! $deleteStatsStmt->execute()) {
echo "***** Error : Deleting statistics $id for non-surveillance node : ". $deleteStatsStmt->error . "\n";
}
}
if (! $deleteTagStmt->execute()) {
echo "***** Error : Deleting tags $id for non-surveillance node : ". $deleteTagStmt->error . "\n";
}

32
data_handler/utils/create_camera_update_statements_download.sh Normal file
View file

@ -0,0 +1,32 @@
#!/bin/bash
exeDir=$(cd "$(dirname "$0")"; pwd)
cd "$exeDir"
REPLICATE_URL=$(grep -oP "REPLICATE_URL[',\s]*['\"]\Khttps?://[^'\"]+" config.php)
echo "Using replicate URL: $REPLICATE_URL"
export REPLICATE_URL
lastSeqNum=$(php get_sync_state.php)
curl -sL "$REPLICATE_URL/state.txt" -o state.txt
newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-)
echo $newSeqNum
if [ "$newSeqNum" -eq "$lastSeqNum" ]; then
echo "No new file to be processed"
exit 0
fi
mkdir -p logs
seq "$((lastSeqNum + 1))" "$newSeqNum" > seq_list.txt
cat seq_list.txt | xargs -n1 -P7 -I{} bash -c '
seqnum="$1"
f=$(printf "%09d" "$seqnum")
path=$(echo "$f" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
url="${REPLICATE_URL}/${path}.osc.gz"
echo "Downloading $url"
curl -sL "$url" -o "${seqnum}.osc.gz" && gunzip "${seqnum}.osc.gz"
' _ {}

24
data_handler/utils/create_camera_update_statements.sh → data_handler/utils/create_camera_update_statements_process.sh
View file

@ -47,21 +47,25 @@ then
fi
mkdir logs
curSeqNum=$lastSeqNum
curSeqNum=$(( $curSeqNum + 1 ))
while [ $curSeqNum -lt $newSeqNum ]
do
curSeqNum=$(( $curSeqNum + 1 ))
if [ -e "$curSeqNum.osc" ]
then
rm "change_file.osc"
cp $curSeqNum.osc change_file.osc
targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'`
targetDirName="$REPLICATE_URL/$targetDirName"
curl -L "$targetDirName" -o change_file.osc.gz
gunzip change_file.osc.gz 2>&1
php create_camera_update_statements.php $curSeqNum 2>&1
php create_camera_update_statements.php $curSeqNum
rm $curSeqNum.osc
curSeqNum=$(( $curSeqNum + 1 ))
else
path=$(echo "$(printf "%09d" "$curSeqNum")" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
url="${REPLICATE_URL}/${path}.osc.gz"
echo $url
curl -sL "$url" -o "$curSeqNum.osc.gz" && gunzip "$curSeqNum.osc.gz"
fi
done
rm "/var/lock/update_camera"