forked from kamba4/sunders
Containerizes application #12
2 changed files with 402 additions and 0 deletions
added create_sql scripts
commit
bf874b83b1
309
data_handler/utils/create_camera_update_statements.php
Executable file
309
data_handler/utils/create_camera_update_statements.php
Executable file
|
|
@ -0,0 +1,309 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
include "config.php";
|
||||||
|
|
||||||
|
$elementTypes = array();
|
||||||
|
$count = 0;
|
||||||
|
$countDelete = 0;
|
||||||
|
$countModifyDelete = 0;
|
||||||
|
$countModify = 0;
|
||||||
|
$countCreate = 0;
|
||||||
|
$mode = '';
|
||||||
|
$curNodeAttrs = null;
|
||||||
|
$curNodeTags = null;
|
||||||
|
$id = 0;
|
||||||
|
$latitude = 0;
|
||||||
|
$longitude = 0;
|
||||||
|
|
||||||
|
$sql_statement = "BEGIN TRANSACTION;\n";
|
||||||
|
|
||||||
|
$mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB);
|
||||||
|
|
||||||
|
if($mysqli->connect_errno) {
|
||||||
|
echo "Error while connecting to DB : $mysqli->error \n" ;
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
$mysqli->autocommit(FALSE);
|
||||||
|
|
||||||
|
if (! ($deleteStmt = $mysqli->prepare("DELETE FROM position WHERE id=?"))) {
|
||||||
|
echo "Error while preparing delete position statement : " . $mysqli->error ;
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
$deleteStmt->bind_param('d', $id);
|
||||||
|
|
||||||
|
if (! ($deleteTagStmt = $mysqli->prepare("DELETE FROM tag WHERE id=?"))) {
|
||||||
|
echo "Error while preparing delete tag statement : " . $mysqli->error ;
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
$deleteTagStmt->bind_param('d', $id);
|
||||||
|
|
||||||
|
if (USE_STATISTICS) {
|
||||||
|
if (! ($deleteStatsStmt = $mysqli->prepare("DELETE FROM statistics WHERE id=?"))) {
|
||||||
|
echo "Error while preparing delete statistics statement : " . $mysqli->error ;
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
$deleteStatsStmt->bind_param('d', $id);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (! ($insertStmt = $mysqli->prepare("INSERT INTO position (id, latitude, longitude) VALUES (?, ?, ?)"))) {
|
||||||
|
echo "Error while preparing insert position statement : " . $mysqli->error ;
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
$insertStmt->bind_param('dii', $id, $latitude, $longitude);
|
||||||
|
|
||||||
|
if (! ($insertTagStmt = $mysqli->prepare("INSERT INTO tag (id, k, v) VALUES (?, ?, ?)"))) {
|
||||||
|
echo "Error while preparing insert tag statement : " . $mysqli->error ;
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
$insertTagStmt->bind_param('dss', $id, $k, $v);
|
||||||
|
|
||||||
|
if (USE_STATISTICS) {
|
||||||
|
if (! ($insertStatsStmt = $mysqli->prepare("INSERT INTO statistics (id, ts, version) VALUES (?, ?, ?)"))) {
|
||||||
|
echo "Error while preparing insert statistics statement : " . $mysqli->error ;
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
$insertStatsStmt->bind_param('dsi', $id, $ts, $version);
|
||||||
|
}
|
||||||
|
|
||||||
|
function printDebug() {
|
||||||
|
global $elementTypes, $count, $countDelete, $countModifyDelete, $countModify, $countCreate;
|
||||||
|
|
||||||
|
echo "== $count ==============================\n";
|
||||||
|
foreach($elementTypes as $k => $v) {
|
||||||
|
echo "$k : $v\n";
|
||||||
|
}
|
||||||
|
echo "++++++ Surveillance nodes : ++++++\n";
|
||||||
|
echo "$countDelete deletions\n";
|
||||||
|
echo "$countModifyDelete modifications --> deletions\n";
|
||||||
|
echo "$countModify modifications\n";
|
||||||
|
echo "$countCreate creations\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
function printDebugCurNode() {
|
||||||
|
global $curNodeAttrs, $curNodeTags, $mode;
|
||||||
|
|
||||||
|
echo "=============\n";
|
||||||
|
echo "$mode : " . $curNodeAttrs['id'] ." (". $curNodeAttrs['lat'] . " x " . $curNodeAttrs['lon'] . ") : " . $curNodeAttrs['user'] . "\n";
|
||||||
|
if (! empty($curNodeAttrs)) {
|
||||||
|
echo " => Attributes :\n";
|
||||||
|
foreach($curNodeAttrs as $k => $v) {
|
||||||
|
echo " $k : $v\n";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
echo " => No attributes\n";
|
||||||
|
}
|
||||||
|
if (! empty($curNodeTags)) {
|
||||||
|
echo " => Tags :\n";
|
||||||
|
foreach($curNodeTags as $k => $v) {
|
||||||
|
echo " $k : $v\n";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
echo " => No tags\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function startElement ($parser, $name, $attrs) {
|
||||||
|
global $elementTypes, $count, $mode, $curNodeAttrs, $curNodeTags;
|
||||||
|
$count++;
|
||||||
|
|
||||||
|
if (array_key_exists($name, $elementTypes)) {
|
||||||
|
$elementTypes[$name] += 1;
|
||||||
|
} else {
|
||||||
|
$elementTypes[$name] = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($name == 'modify' || $name == 'delete' or $name == 'create') {
|
||||||
|
$mode = $name;
|
||||||
|
|
||||||
|
} else if ($name == 'node') {
|
||||||
|
$curNodeAttrs = $attrs;
|
||||||
|
$curNodeTags = array();
|
||||||
|
|
||||||
|
} else if ($name == 'tag') {
|
||||||
|
$curNodeTags[$attrs['k']] = $attrs['v'];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function endElement ($parser, $name) {
|
||||||
|
if (USE_STATISTICS) {
|
||||||
|
global $mode, $deleteStmt, $deleteTagStmt, $deleteStatsStmt, $insertStmt, $insertTagStmt, $insertStatsStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $ts, $version, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate, $sql_statement;
|
||||||
|
} else {
|
||||||
|
global $mode, $deleteStmt, $deleteTagStmt, $insertStmt, $insertTagStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate, $sql_statement;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($name == 'modify' || $name == 'delete' or $name == 'create') {
|
||||||
|
$mode = '';
|
||||||
|
} else if ($name == 'node') {
|
||||||
|
if ($mode == 'delete') {
|
||||||
|
$id = $curNodeAttrs['id'];
|
||||||
|
|
||||||
|
if (USE_STATISTICS) {
|
||||||
|
if (! $deleteStatsStmt->execute()) {
|
||||||
|
echo "***** Error : Deleting statistics $id : ". $deleteStatsStmt->error . "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (! $deleteTagStmt->execute()) {
|
||||||
|
echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (! $deleteStmt->execute()) {
|
||||||
|
echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n";
|
||||||
|
}
|
||||||
|
if ($deleteStmt->affected_rows > 0) {
|
||||||
|
$sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
|
||||||
|
$sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
|
||||||
|
$countDelete++;
|
||||||
|
$mysqli->commit();
|
||||||
|
printDebugCurNode();
|
||||||
|
}
|
||||||
|
|
||||||
|
} else if ($mode == 'modify' || $mode == 'create') {
|
||||||
|
if (! empty($curNodeTags)
|
||||||
|
&& array_key_exists('man_made', $curNodeTags)
|
||||||
|
&& $curNodeTags['man_made'] == 'surveillance') {
|
||||||
|
|
||||||
|
printDebugCurNode();
|
||||||
|
|
||||||
|
$id = $curNodeAttrs['id'];
|
||||||
|
|
||||||
|
if ($mode == 'modify') {
|
||||||
|
$countModify++;
|
||||||
|
|
||||||
|
if (USE_STATISTICS) {
|
||||||
|
if (! $deleteStatsStmt->execute()) {
|
||||||
|
echo "***** Error : Deleting statistics $id for modification : ". $deleteStatsStmt->error . "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (! $deleteTagStmt->execute()) {
|
||||||
|
echo "***** Error : Deleting tags $id for modification : ". $deleteTagStmt->error . "\n";
|
||||||
|
}
|
||||||
|
if (! $deleteStmt->execute()) {
|
||||||
|
echo "***** Error : Deleting $id for modification : ". $deleteStmt->error . "\n";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$countCreate++;
|
||||||
|
}
|
||||||
|
|
||||||
|
$latitude = (int) ($curNodeAttrs['lat'] * 10000000);
|
||||||
|
$longitude = (int) ($curNodeAttrs['lon'] * 10000000);
|
||||||
|
$sql_statement = $sql_statement . "INSERT INTO position (id, latitude, longitude) VALUES ($id, $latitude , $longitude);\n";
|
||||||
|
if (! $insertStmt->execute()) {
|
||||||
|
echo "***** Error : inserting $id ($latitude x $longitude) : ". $insertStmt->error . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
$k = 'lat';
|
||||||
|
$v = $curNodeAttrs['lat'];
|
||||||
|
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||||
|
if (! $insertTagStmt->execute()) {
|
||||||
|
echo "***** Error : inserting latitude $v for $id : ". $insertTagStmt->error . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
$k = 'lon';
|
||||||
|
$v = $curNodeAttrs['lon'];
|
||||||
|
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||||
|
if (! $insertTagStmt->execute()) {
|
||||||
|
echo "***** Error : inserting longitude $v for $id : ". $insertTagStmt->error . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
$k = 'userid';
|
||||||
|
$v = $curNodeAttrs['user'];
|
||||||
|
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||||
|
|
||||||
|
if (! $insertTagStmt->execute()) {
|
||||||
|
echo "***** Error : inserting user $v for $id : ". $insertTagStmt->error . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
$k = 'version';
|
||||||
|
$v = $curNodeAttrs['version'];
|
||||||
|
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||||
|
if (! $insertTagStmt->execute()) {
|
||||||
|
echo "***** Error : inserting version $v for $id : ". $insertTagStmt->error . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
$k = 'timestamp';
|
||||||
|
$v = $curNodeAttrs['timestamp'];
|
||||||
|
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||||
|
if (! $insertTagStmt->execute()) {
|
||||||
|
echo "***** Error : inserting timestamp $v for $id : ". $insertTagStmt->error . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach($curNodeTags as $k => $v) {
|
||||||
|
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||||
|
if (! $insertTagStmt->execute()) {
|
||||||
|
echo "***** Error : inserting tag $k => $v for $id : ". $insertTagStmt->error . "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (USE_STATISTICS) {
|
||||||
|
$ts = $curNodeAttrs['timestamp'];
|
||||||
|
$version = $curNodeAttrs['version'];
|
||||||
|
if (! $insertStatsStmt->execute()) {
|
||||||
|
echo "***** Error : inserting ts $ts, version $version for $id : ". $insertStatsStmt->error . "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$mysqli->commit();
|
||||||
|
} else if ($mode == 'modify') {
|
||||||
|
// delete former surveillance nodes that were modified to non-surveillance nodes
|
||||||
|
$id = $curNodeAttrs['id'];
|
||||||
|
|
||||||
|
if (USE_STATISTICS) {
|
||||||
|
if (! $deleteStatsStmt->execute()) {
|
||||||
|
echo "***** Error : Deleting statistics $id for non-surveillance node : ". $deleteStatsStmt->error . "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (! $deleteTagStmt->execute()) {
|
||||||
|
echo "***** Error : Deleting tags $id for non-surveillance node : ". $deleteTagStmt->error . "\n";
|
||||||
|
}
|
||||||
|
if (! $deleteStmt->execute()) {
|
||||||
|
echo "***** Error : Deleting $id for non-surveillance node : ". $deleteStmt->error . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($deleteStmt->affected_rows > 0) {
|
||||||
|
$sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
|
||||||
|
$sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
|
||||||
|
$countModifyDelete++;
|
||||||
|
$mysqli->commit();
|
||||||
|
printDebugCurNode();
|
||||||
|
echo " ==> delete because modified to non-surveillance node\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$curNodeAttrs = null;
|
||||||
|
$curNodeTags = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$file="change_file.osc";
|
||||||
|
|
||||||
|
$xml_parser = xml_parser_create();
|
||||||
|
xml_parser_set_option($xml_parser, XML_OPTION_CASE_FOLDING, false);
|
||||||
|
xml_set_element_handler($xml_parser, "startElement", "endElement");
|
||||||
|
|
||||||
|
if (!($fp = fopen($file, "r"))) {
|
||||||
|
die("could not open XML input");
|
||||||
|
}
|
||||||
|
|
||||||
|
while ($data = fread($fp, 4096)) {
|
||||||
|
if (!xml_parse($xml_parser, $data, feof($fp))) {
|
||||||
|
die(sprintf("XML error: %s at line %d",
|
||||||
|
xml_error_string(xml_get_error_code($xml_parser)),
|
||||||
|
xml_get_current_line_number($xml_parser)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$sql="UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';";
|
||||||
|
$mysqli->query($sql);
|
||||||
|
$mysqli->commit();
|
||||||
|
$sql_statement = $sql_statement . "UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';\n";
|
||||||
|
$sql_statement = $sql_statement . "COMMIT;";
|
||||||
|
$sql_file = "/migrations/".$_SERVER['argv'][1].".sql";
|
||||||
|
|
||||||
|
file_put_contents($sql_file, $sql_statement);
|
||||||
|
printDebug();
|
||||||
|
$mysqli->close();
|
||||||
|
|
||||||
|
?>
|
||||||
93
data_handler/utils/create_camera_update_statements.sh
Normal file
93
data_handler/utils/create_camera_update_statements.sh
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
exedir=$(cd `dirname "$0"`; pwd)
|
||||||
|
|
||||||
|
REPLICATE_URL=`grep "REPLICATE_URL" "$exedir/config.php" | sed -e 's/.*http/http/' -e 's#...$##' `
|
||||||
|
|
||||||
|
if [ -e "/var/lock/update_camera" ]
|
||||||
|
then
|
||||||
|
# Maybe an other update is running
|
||||||
|
otherPid=$(cat "/var/lock/update_camera")
|
||||||
|
count=$(ps $otherPid | grep -c `basename "$0"`)
|
||||||
|
|
||||||
|
if [ $count -gt 0 ]
|
||||||
|
then
|
||||||
|
echo "$0 is running yet. Exiting." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# OK. We can update the database...
|
||||||
|
echo $$ > "/var/lock/update_camera"
|
||||||
|
|
||||||
|
exeDir=$(cd `dirname "$0"`; pwd);
|
||||||
|
cd "$exeDir"
|
||||||
|
|
||||||
|
if [ -e "state.txt" ]
|
||||||
|
then
|
||||||
|
rm "state.txt"
|
||||||
|
fi
|
||||||
|
|
||||||
|
#TODO get state from db
|
||||||
|
# Read the last update timestamp
|
||||||
|
# lastTimestamp=$(grep "^timestamp=" "lastState.txt" | cut -d'=' -f2-)
|
||||||
|
# lastSeqNum=$(grep "^sequenceNumber=" "lastState.txt" | cut -d'=' -f2-)
|
||||||
|
|
||||||
|
lastSeqNum=$(php get_sync_state.php)
|
||||||
|
|
||||||
|
curl -L "$REPLICATE_URL/state.txt" -o state.txt
|
||||||
|
|
||||||
|
newTimestamp=$(grep "^timestamp=" "state.txt" | cut -d'=' -f2-)
|
||||||
|
newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-)
|
||||||
|
|
||||||
|
if [ $newSeqNum -eq $lastSeqNum ]
|
||||||
|
then
|
||||||
|
echo "No new file to be processed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
mkdir logs
|
||||||
|
curSeqNum=$lastSeqNum
|
||||||
|
while [ $curSeqNum -lt $newSeqNum ]
|
||||||
|
do
|
||||||
|
curSeqNum=$(( $curSeqNum + 1 ))
|
||||||
|
|
||||||
|
logFileName="logs/log.$curSeqNum"
|
||||||
|
|
||||||
|
if [ -e "change_file.osc" ]
|
||||||
|
then
|
||||||
|
rm "change_file.osc"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -e "change_file.osc.gz" ]
|
||||||
|
then
|
||||||
|
rm "change_file.osc.gz"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName"
|
||||||
|
|
||||||
|
targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'`
|
||||||
|
targetDirName="$REPLICATE_URL/$targetDirName"
|
||||||
|
|
||||||
|
curl -L "$targetDirName" -o change_file.osc.gz 2>&1 | tee -a $logFileName
|
||||||
|
if [ $? -gt 0 ]
|
||||||
|
then
|
||||||
|
echo "Error during recovery of $targetDirName" | tee -a $logFileName
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
gunzip change_file.osc.gz 2>&1 | tee -a $logFileName
|
||||||
|
if [ $? -gt 0 ]
|
||||||
|
then
|
||||||
|
echo "Error during decompression of $targetDirName" | tee -a $logFileName
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
php create_camera_update_statements.php $curSeqNum 2>&1 | tee -a $logFileName
|
||||||
|
|
||||||
|
targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'`
|
||||||
|
|
||||||
|
curl -L "$targetDirName" -o lastState.txt 2>&1 | tee -a $logFileName
|
||||||
|
echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName"
|
||||||
|
done
|
||||||
|
|
||||||
|
rm "/var/lock/update_camera"
|
||||||
Loading…
Add table
Add a link
Reference in a new issue