diff --git a/data_handler/utils/create_camera_update_statements.php b/data_handler/utils/create_camera_update_statements.php index 57962c8..93adb05 100755 --- a/data_handler/utils/create_camera_update_statements.php +++ b/data_handler/utils/create_camera_update_statements.php @@ -57,50 +57,6 @@ if (! ($insertTagStmt = $mysqli->prepare("INSERT INTO tag (id, k, v) VALUES (?, } $insertTagStmt->bind_param('dss', $id, $k, $v); -if (USE_STATISTICS) { - if (! ($insertStatsStmt = $mysqli->prepare("INSERT INTO statistics (id, ts, version) VALUES (?, ?, ?)"))) { - echo "Error while preparing insert statistics statement : " . $mysqli->error ; - exit(1); - } - $insertStatsStmt->bind_param('dsi', $id, $ts, $version); -} - -function printDebug() { - global $elementTypes, $count, $countDelete, $countModifyDelete, $countModify, $countCreate; - - echo "== $count ==============================\n"; - foreach($elementTypes as $k => $v) { - echo "$k : $v\n"; - } - echo "++++++ Surveillance nodes : ++++++\n"; - echo "$countDelete deletions\n"; - echo "$countModifyDelete modifications --> deletions\n"; - echo "$countModify modifications\n"; - echo "$countCreate creations\n"; -} - -function printDebugCurNode() { - global $curNodeAttrs, $curNodeTags, $mode; - - echo "=============\n"; - echo "$mode : " . $curNodeAttrs['id'] ." (". $curNodeAttrs['lat'] . " x " . $curNodeAttrs['lon'] . ") : " . $curNodeAttrs['user'] . "\n"; - if (! empty($curNodeAttrs)) { - echo " => Attributes :\n"; - foreach($curNodeAttrs as $k => $v) { - echo " $k : $v\n"; - } - } else { - echo " => No attributes\n"; - } - if (! empty($curNodeTags)) { - echo " => Tags :\n"; - foreach($curNodeTags as $k => $v) { - echo " $k : $v\n"; - } - } else { - echo " => No tags\n"; - } -} function startElement ($parser, $name, $attrs) { global $elementTypes, $count, $mode, $curNodeAttrs, $curNodeTags; @@ -137,12 +93,6 @@ function endElement ($parser, $name) { if ($mode == 'delete') { $id = $curNodeAttrs['id']; - if (USE_STATISTICS) { - if (! $deleteStatsStmt->execute()) { - echo "***** Error : Deleting statistics $id : ". $deleteStatsStmt->error . "\n"; - } - } - if (! $deleteTagStmt->execute()) { echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n"; } @@ -151,6 +101,13 @@ function endElement ($parser, $name) { echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n"; } if ($deleteStmt->affected_rows > 0) { + if (! $deleteTagStmt->execute()) { + echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n"; + } + + if (! $deleteStmt->execute()) { + echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n"; + } $sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n"; $sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n"; } @@ -165,19 +122,14 @@ function endElement ($parser, $name) { $id = $curNodeAttrs['id']; if ($mode == 'modify') { - - if (USE_STATISTICS) { - if (! $deleteStatsStmt->execute()) { - echo "***** Error : Deleting statistics $id for modification : ". $deleteStatsStmt->error . "\n"; - } - } - if (! $deleteTagStmt->execute()) { echo "***** Error : Deleting tags $id for modification : ". $deleteTagStmt->error . "\n"; } if (! $deleteStmt->execute()) { echo "***** Error : Deleting $id for modification : ". $deleteStmt->error . "\n"; } + $sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n"; + $sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n"; } else { } @@ -198,57 +150,30 @@ function endElement ($parser, $name) { $k = 'lon'; $v = $curNodeAttrs['lon']; $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; - if (! $insertTagStmt->execute()) { - echo "***** Error : inserting longitude $v for $id : ". $insertTagStmt->error . "\n"; - } $k = 'userid'; $v = $curNodeAttrs['user']; $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; - if (! $insertTagStmt->execute()) { - echo "***** Error : inserting user $v for $id : ". $insertTagStmt->error . "\n"; - } $k = 'version'; $v = $curNodeAttrs['version']; $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; - if (! $insertTagStmt->execute()) { - echo "***** Error : inserting version $v for $id : ". $insertTagStmt->error . "\n"; - } + $k = 'timestamp'; $v = $curNodeAttrs['timestamp']; $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; - if (! $insertTagStmt->execute()) { - echo "***** Error : inserting timestamp $v for $id : ". $insertTagStmt->error . "\n"; - } + foreach($curNodeTags as $k => $v) { $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; - if (! $insertTagStmt->execute()) { - echo "***** Error : inserting tag $k => $v for $id : ". $insertTagStmt->error . "\n"; - } + } - - if (USE_STATISTICS) { - $ts = $curNodeAttrs['timestamp']; - $version = $curNodeAttrs['version']; - if (! $insertStatsStmt->execute()) { - echo "***** Error : inserting ts $ts, version $version for $id : ". $insertStatsStmt->error . "\n"; - } - } - - $mysqli->commit(); } else if ($mode == 'modify') { // delete former surveillance nodes that were modified to non-surveillance nodes $id = $curNodeAttrs['id']; - if (USE_STATISTICS) { - if (! $deleteStatsStmt->execute()) { - echo "***** Error : Deleting statistics $id for non-surveillance node : ". $deleteStatsStmt->error . "\n"; - } - } if (! $deleteTagStmt->execute()) { echo "***** Error : Deleting tags $id for non-surveillance node : ". $deleteTagStmt->error . "\n"; } diff --git a/data_handler/utils/create_camera_update_statements_download.sh b/data_handler/utils/create_camera_update_statements_download.sh new file mode 100644 index 0000000..17ccdbb --- /dev/null +++ b/data_handler/utils/create_camera_update_statements_download.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +exeDir=$(cd "$(dirname "$0")"; pwd) +cd "$exeDir" + +REPLICATE_URL=$(grep -oP "REPLICATE_URL[',\s]*['\"]\Khttps?://[^'\"]+" config.php) + +echo "Using replicate URL: $REPLICATE_URL" +export REPLICATE_URL + + +lastSeqNum=$(php get_sync_state.php) +curl -sL "$REPLICATE_URL/state.txt" -o state.txt +newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-) +echo $newSeqNum +if [ "$newSeqNum" -eq "$lastSeqNum" ]; then + echo "No new file to be processed" + exit 0 +fi + +mkdir -p logs +seq "$((lastSeqNum + 1))" "$newSeqNum" > seq_list.txt + + +cat seq_list.txt | xargs -n1 -P7 -I{} bash -c ' + seqnum="$1" + f=$(printf "%09d" "$seqnum") + path=$(echo "$f" | sed -E "s#(...)(...)(...)#\1/\2/\3#") + url="${REPLICATE_URL}/${path}.osc.gz" + echo "Downloading $url" + curl -sL "$url" -o "${seqnum}.osc.gz" && gunzip "${seqnum}.osc.gz" +' _ {} diff --git a/data_handler/utils/create_camera_update_statements.sh b/data_handler/utils/create_camera_update_statements_process.sh similarity index 75% rename from data_handler/utils/create_camera_update_statements.sh rename to data_handler/utils/create_camera_update_statements_process.sh index b306052..87267e9 100644 --- a/data_handler/utils/create_camera_update_statements.sh +++ b/data_handler/utils/create_camera_update_statements_process.sh @@ -47,21 +47,25 @@ then fi mkdir logs curSeqNum=$lastSeqNum +curSeqNum=$(( $curSeqNum + 1 )) while [ $curSeqNum -lt $newSeqNum ] do - curSeqNum=$(( $curSeqNum + 1 )) + if [ -e "$curSeqNum.osc" ] + then + rm "change_file.osc" + cp $curSeqNum.osc change_file.osc - - targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'` - targetDirName="$REPLICATE_URL/$targetDirName" - - curl -L "$targetDirName" -o change_file.osc.gz - - gunzip change_file.osc.gz 2>&1 - - php create_camera_update_statements.php $curSeqNum 2>&1 + php create_camera_update_statements.php $curSeqNum + rm $curSeqNum.osc + curSeqNum=$(( $curSeqNum + 1 )) + else + path=$(echo "$(printf "%09d" "$curSeqNum")" | sed -E "s#(...)(...)(...)#\1/\2/\3#") + url="${REPLICATE_URL}/${path}.osc.gz" + echo $url + curl -sL "$url" -o "$curSeqNum.osc.gz" && gunzip "$curSeqNum.osc.gz" + fi done rm "/var/lock/update_camera"