diff --git a/data_handler/utils/create_camera_update_statements.php b/data_handler/utils/create_camera_update_statements.php new file mode 100755 index 0000000..c3d056f --- /dev/null +++ b/data_handler/utils/create_camera_update_statements.php @@ -0,0 +1,309 @@ +connect_errno) { + echo "Error while connecting to DB : $mysqli->error \n" ; + exit(1); +} +$mysqli->autocommit(FALSE); + +if (! ($deleteStmt = $mysqli->prepare("DELETE FROM position WHERE id=?"))) { + echo "Error while preparing delete position statement : " . $mysqli->error ; + exit(1); +} +$deleteStmt->bind_param('d', $id); + +if (! ($deleteTagStmt = $mysqli->prepare("DELETE FROM tag WHERE id=?"))) { + echo "Error while preparing delete tag statement : " . $mysqli->error ; + exit(1); +} +$deleteTagStmt->bind_param('d', $id); + +if (USE_STATISTICS) { + if (! ($deleteStatsStmt = $mysqli->prepare("DELETE FROM statistics WHERE id=?"))) { + echo "Error while preparing delete statistics statement : " . $mysqli->error ; + exit(1); + } + $deleteStatsStmt->bind_param('d', $id); +} + +if (! ($insertStmt = $mysqli->prepare("INSERT INTO position (id, latitude, longitude) VALUES (?, ?, ?)"))) { + echo "Error while preparing insert position statement : " . $mysqli->error ; + exit(1); +} +$insertStmt->bind_param('dii', $id, $latitude, $longitude); + +if (! ($insertTagStmt = $mysqli->prepare("INSERT INTO tag (id, k, v) VALUES (?, ?, ?)"))) { + echo "Error while preparing insert tag statement : " . $mysqli->error ; + exit(1); +} +$insertTagStmt->bind_param('dss', $id, $k, $v); + +if (USE_STATISTICS) { + if (! ($insertStatsStmt = $mysqli->prepare("INSERT INTO statistics (id, ts, version) VALUES (?, ?, ?)"))) { + echo "Error while preparing insert statistics statement : " . $mysqli->error ; + exit(1); + } + $insertStatsStmt->bind_param('dsi', $id, $ts, $version); +} + +function printDebug() { + global $elementTypes, $count, $countDelete, $countModifyDelete, $countModify, $countCreate; + + echo "== $count ==============================\n"; + foreach($elementTypes as $k => $v) { + echo "$k : $v\n"; + } + echo "++++++ Surveillance nodes : ++++++\n"; + echo "$countDelete deletions\n"; + echo "$countModifyDelete modifications --> deletions\n"; + echo "$countModify modifications\n"; + echo "$countCreate creations\n"; +} + +function printDebugCurNode() { + global $curNodeAttrs, $curNodeTags, $mode; + + echo "=============\n"; + echo "$mode : " . $curNodeAttrs['id'] ." (". $curNodeAttrs['lat'] . " x " . $curNodeAttrs['lon'] . ") : " . $curNodeAttrs['user'] . "\n"; + if (! empty($curNodeAttrs)) { + echo " => Attributes :\n"; + foreach($curNodeAttrs as $k => $v) { + echo " $k : $v\n"; + } + } else { + echo " => No attributes\n"; + } + if (! empty($curNodeTags)) { + echo " => Tags :\n"; + foreach($curNodeTags as $k => $v) { + echo " $k : $v\n"; + } + } else { + echo " => No tags\n"; + } +} + +function startElement ($parser, $name, $attrs) { + global $elementTypes, $count, $mode, $curNodeAttrs, $curNodeTags; + $count++; + + if (array_key_exists($name, $elementTypes)) { + $elementTypes[$name] += 1; + } else { + $elementTypes[$name] = 1; + } + + if ($name == 'modify' || $name == 'delete' or $name == 'create') { + $mode = $name; + + } else if ($name == 'node') { + $curNodeAttrs = $attrs; + $curNodeTags = array(); + + } else if ($name == 'tag') { + $curNodeTags[$attrs['k']] = $attrs['v']; + } +} + +function endElement ($parser, $name) { + if (USE_STATISTICS) { + global $mode, $deleteStmt, $deleteTagStmt, $deleteStatsStmt, $insertStmt, $insertTagStmt, $insertStatsStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $ts, $version, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate, $sql_statement; + } else { + global $mode, $deleteStmt, $deleteTagStmt, $insertStmt, $insertTagStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate, $sql_statement; + } + + if ($name == 'modify' || $name == 'delete' or $name == 'create') { + $mode = ''; + } else if ($name == 'node') { + if ($mode == 'delete') { + $id = $curNodeAttrs['id']; + + if (USE_STATISTICS) { + if (! $deleteStatsStmt->execute()) { + echo "***** Error : Deleting statistics $id : ". $deleteStatsStmt->error . "\n"; + } + } + + if (! $deleteTagStmt->execute()) { + echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n"; + } + + if (! $deleteStmt->execute()) { + echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n"; + } + if ($deleteStmt->affected_rows > 0) { + $sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n"; + $sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n"; + $countDelete++; + $mysqli->commit(); + printDebugCurNode(); + } + + } else if ($mode == 'modify' || $mode == 'create') { + if (! empty($curNodeTags) + && array_key_exists('man_made', $curNodeTags) + && $curNodeTags['man_made'] == 'surveillance') { + + printDebugCurNode(); + + $id = $curNodeAttrs['id']; + + if ($mode == 'modify') { + $countModify++; + + if (USE_STATISTICS) { + if (! $deleteStatsStmt->execute()) { + echo "***** Error : Deleting statistics $id for modification : ". $deleteStatsStmt->error . "\n"; + } + } + + if (! $deleteTagStmt->execute()) { + echo "***** Error : Deleting tags $id for modification : ". $deleteTagStmt->error . "\n"; + } + if (! $deleteStmt->execute()) { + echo "***** Error : Deleting $id for modification : ". $deleteStmt->error . "\n"; + } + } else { + $countCreate++; + } + + $latitude = (int) ($curNodeAttrs['lat'] * 10000000); + $longitude = (int) ($curNodeAttrs['lon'] * 10000000); + $sql_statement = $sql_statement . "INSERT INTO position (id, latitude, longitude) VALUES ($id, $latitude , $longitude);\n"; + if (! $insertStmt->execute()) { + echo "***** Error : inserting $id ($latitude x $longitude) : ". $insertStmt->error . "\n"; + } + + $k = 'lat'; + $v = $curNodeAttrs['lat']; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + if (! $insertTagStmt->execute()) { + echo "***** Error : inserting latitude $v for $id : ". $insertTagStmt->error . "\n"; + } + + $k = 'lon'; + $v = $curNodeAttrs['lon']; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + if (! $insertTagStmt->execute()) { + echo "***** Error : inserting longitude $v for $id : ". $insertTagStmt->error . "\n"; + } + + $k = 'userid'; + $v = $curNodeAttrs['user']; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + + if (! $insertTagStmt->execute()) { + echo "***** Error : inserting user $v for $id : ". $insertTagStmt->error . "\n"; + } + + $k = 'version'; + $v = $curNodeAttrs['version']; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + if (! $insertTagStmt->execute()) { + echo "***** Error : inserting version $v for $id : ". $insertTagStmt->error . "\n"; + } + + $k = 'timestamp'; + $v = $curNodeAttrs['timestamp']; + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + if (! $insertTagStmt->execute()) { + echo "***** Error : inserting timestamp $v for $id : ". $insertTagStmt->error . "\n"; + } + + foreach($curNodeTags as $k => $v) { + $sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n"; + if (! $insertTagStmt->execute()) { + echo "***** Error : inserting tag $k => $v for $id : ". $insertTagStmt->error . "\n"; + } + } + + if (USE_STATISTICS) { + $ts = $curNodeAttrs['timestamp']; + $version = $curNodeAttrs['version']; + if (! $insertStatsStmt->execute()) { + echo "***** Error : inserting ts $ts, version $version for $id : ". $insertStatsStmt->error . "\n"; + } + } + + $mysqli->commit(); + } else if ($mode == 'modify') { + // delete former surveillance nodes that were modified to non-surveillance nodes + $id = $curNodeAttrs['id']; + + if (USE_STATISTICS) { + if (! $deleteStatsStmt->execute()) { + echo "***** Error : Deleting statistics $id for non-surveillance node : ". $deleteStatsStmt->error . "\n"; + } + } + if (! $deleteTagStmt->execute()) { + echo "***** Error : Deleting tags $id for non-surveillance node : ". $deleteTagStmt->error . "\n"; + } + if (! $deleteStmt->execute()) { + echo "***** Error : Deleting $id for non-surveillance node : ". $deleteStmt->error . "\n"; + } + + if ($deleteStmt->affected_rows > 0) { + $sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n"; + $sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n"; + $countModifyDelete++; + $mysqli->commit(); + printDebugCurNode(); + echo " ==> delete because modified to non-surveillance node\n"; + } + } + } + + $curNodeAttrs = null; + $curNodeTags = null; + } +} + +$file="change_file.osc"; + +$xml_parser = xml_parser_create(); +xml_parser_set_option($xml_parser, XML_OPTION_CASE_FOLDING, false); +xml_set_element_handler($xml_parser, "startElement", "endElement"); + +if (!($fp = fopen($file, "r"))) { + die("could not open XML input"); +} + +while ($data = fread($fp, 4096)) { + if (!xml_parse($xml_parser, $data, feof($fp))) { + die(sprintf("XML error: %s at line %d", + xml_error_string(xml_get_error_code($xml_parser)), + xml_get_current_line_number($xml_parser))); + } +} + +$sql="UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';"; +$mysqli->query($sql); +$mysqli->commit(); +$sql_statement = $sql_statement . "UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';\n"; +$sql_statement = $sql_statement . "COMMIT;"; +$sql_file = "/migrations/".$_SERVER['argv'][1].".sql"; + +file_put_contents($sql_file, $sql_statement); +printDebug(); +$mysqli->close(); + +?> diff --git a/data_handler/utils/create_camera_update_statements.sh b/data_handler/utils/create_camera_update_statements.sh new file mode 100644 index 0000000..a9fa465 --- /dev/null +++ b/data_handler/utils/create_camera_update_statements.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +exedir=$(cd `dirname "$0"`; pwd) + +REPLICATE_URL=`grep "REPLICATE_URL" "$exedir/config.php" | sed -e 's/.*http/http/' -e 's#...$##' ` + +if [ -e "/var/lock/update_camera" ] +then + # Maybe an other update is running + otherPid=$(cat "/var/lock/update_camera") + count=$(ps $otherPid | grep -c `basename "$0"`) + + if [ $count -gt 0 ] + then + echo "$0 is running yet. Exiting." >&2 + exit 1 + fi +fi + +# OK. We can update the database... +echo $$ > "/var/lock/update_camera" + +exeDir=$(cd `dirname "$0"`; pwd); +cd "$exeDir" + +if [ -e "state.txt" ] +then + rm "state.txt" +fi + +#TODO get state from db +# Read the last update timestamp +# lastTimestamp=$(grep "^timestamp=" "lastState.txt" | cut -d'=' -f2-) +# lastSeqNum=$(grep "^sequenceNumber=" "lastState.txt" | cut -d'=' -f2-) + +lastSeqNum=$(php get_sync_state.php) + +curl -L "$REPLICATE_URL/state.txt" -o state.txt + +newTimestamp=$(grep "^timestamp=" "state.txt" | cut -d'=' -f2-) +newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-) + +if [ $newSeqNum -eq $lastSeqNum ] +then + echo "No new file to be processed" + exit 1 +fi +mkdir logs +curSeqNum=$lastSeqNum +while [ $curSeqNum -lt $newSeqNum ] +do + curSeqNum=$(( $curSeqNum + 1 )) + + logFileName="logs/log.$curSeqNum" + + if [ -e "change_file.osc" ] + then + rm "change_file.osc" + fi + + if [ -e "change_file.osc.gz" ] + then + rm "change_file.osc.gz" + fi + + echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName" + + targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'` + targetDirName="$REPLICATE_URL/$targetDirName" + + curl -L "$targetDirName" -o change_file.osc.gz 2>&1 | tee -a $logFileName + if [ $? -gt 0 ] + then + echo "Error during recovery of $targetDirName" | tee -a $logFileName + exit 1 + fi + + gunzip change_file.osc.gz 2>&1 | tee -a $logFileName + if [ $? -gt 0 ] + then + echo "Error during decompression of $targetDirName" | tee -a $logFileName + exit 1 + fi + + php create_camera_update_statements.php $curSeqNum 2>&1 | tee -a $logFileName + + targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'` + + curl -L "$targetDirName" -o lastState.txt 2>&1 | tee -a $logFileName + echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName" +done + +rm "/var/lock/update_camera"