forked from kamba4/sunders
added create_sql scripts
This commit is contained in:
parent
8d5b74a773
commit
bf874b83b1
2 changed files with 402 additions and 0 deletions
309
data_handler/utils/create_camera_update_statements.php
Executable file
309
data_handler/utils/create_camera_update_statements.php
Executable file
|
|
@ -0,0 +1,309 @@
|
|||
<?php
|
||||
|
||||
include "config.php";
|
||||
|
||||
$elementTypes = array();
|
||||
$count = 0;
|
||||
$countDelete = 0;
|
||||
$countModifyDelete = 0;
|
||||
$countModify = 0;
|
||||
$countCreate = 0;
|
||||
$mode = '';
|
||||
$curNodeAttrs = null;
|
||||
$curNodeTags = null;
|
||||
$id = 0;
|
||||
$latitude = 0;
|
||||
$longitude = 0;
|
||||
|
||||
$sql_statement = "BEGIN TRANSACTION;\n";
|
||||
|
||||
$mysqli = new mysqli(MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD, MYSQL_DB);
|
||||
|
||||
if($mysqli->connect_errno) {
|
||||
echo "Error while connecting to DB : $mysqli->error \n" ;
|
||||
exit(1);
|
||||
}
|
||||
$mysqli->autocommit(FALSE);
|
||||
|
||||
if (! ($deleteStmt = $mysqli->prepare("DELETE FROM position WHERE id=?"))) {
|
||||
echo "Error while preparing delete position statement : " . $mysqli->error ;
|
||||
exit(1);
|
||||
}
|
||||
$deleteStmt->bind_param('d', $id);
|
||||
|
||||
if (! ($deleteTagStmt = $mysqli->prepare("DELETE FROM tag WHERE id=?"))) {
|
||||
echo "Error while preparing delete tag statement : " . $mysqli->error ;
|
||||
exit(1);
|
||||
}
|
||||
$deleteTagStmt->bind_param('d', $id);
|
||||
|
||||
if (USE_STATISTICS) {
|
||||
if (! ($deleteStatsStmt = $mysqli->prepare("DELETE FROM statistics WHERE id=?"))) {
|
||||
echo "Error while preparing delete statistics statement : " . $mysqli->error ;
|
||||
exit(1);
|
||||
}
|
||||
$deleteStatsStmt->bind_param('d', $id);
|
||||
}
|
||||
|
||||
if (! ($insertStmt = $mysqli->prepare("INSERT INTO position (id, latitude, longitude) VALUES (?, ?, ?)"))) {
|
||||
echo "Error while preparing insert position statement : " . $mysqli->error ;
|
||||
exit(1);
|
||||
}
|
||||
$insertStmt->bind_param('dii', $id, $latitude, $longitude);
|
||||
|
||||
if (! ($insertTagStmt = $mysqli->prepare("INSERT INTO tag (id, k, v) VALUES (?, ?, ?)"))) {
|
||||
echo "Error while preparing insert tag statement : " . $mysqli->error ;
|
||||
exit(1);
|
||||
}
|
||||
$insertTagStmt->bind_param('dss', $id, $k, $v);
|
||||
|
||||
if (USE_STATISTICS) {
|
||||
if (! ($insertStatsStmt = $mysqli->prepare("INSERT INTO statistics (id, ts, version) VALUES (?, ?, ?)"))) {
|
||||
echo "Error while preparing insert statistics statement : " . $mysqli->error ;
|
||||
exit(1);
|
||||
}
|
||||
$insertStatsStmt->bind_param('dsi', $id, $ts, $version);
|
||||
}
|
||||
|
||||
function printDebug() {
|
||||
global $elementTypes, $count, $countDelete, $countModifyDelete, $countModify, $countCreate;
|
||||
|
||||
echo "== $count ==============================\n";
|
||||
foreach($elementTypes as $k => $v) {
|
||||
echo "$k : $v\n";
|
||||
}
|
||||
echo "++++++ Surveillance nodes : ++++++\n";
|
||||
echo "$countDelete deletions\n";
|
||||
echo "$countModifyDelete modifications --> deletions\n";
|
||||
echo "$countModify modifications\n";
|
||||
echo "$countCreate creations\n";
|
||||
}
|
||||
|
||||
function printDebugCurNode() {
|
||||
global $curNodeAttrs, $curNodeTags, $mode;
|
||||
|
||||
echo "=============\n";
|
||||
echo "$mode : " . $curNodeAttrs['id'] ." (". $curNodeAttrs['lat'] . " x " . $curNodeAttrs['lon'] . ") : " . $curNodeAttrs['user'] . "\n";
|
||||
if (! empty($curNodeAttrs)) {
|
||||
echo " => Attributes :\n";
|
||||
foreach($curNodeAttrs as $k => $v) {
|
||||
echo " $k : $v\n";
|
||||
}
|
||||
} else {
|
||||
echo " => No attributes\n";
|
||||
}
|
||||
if (! empty($curNodeTags)) {
|
||||
echo " => Tags :\n";
|
||||
foreach($curNodeTags as $k => $v) {
|
||||
echo " $k : $v\n";
|
||||
}
|
||||
} else {
|
||||
echo " => No tags\n";
|
||||
}
|
||||
}
|
||||
|
||||
function startElement ($parser, $name, $attrs) {
|
||||
global $elementTypes, $count, $mode, $curNodeAttrs, $curNodeTags;
|
||||
$count++;
|
||||
|
||||
if (array_key_exists($name, $elementTypes)) {
|
||||
$elementTypes[$name] += 1;
|
||||
} else {
|
||||
$elementTypes[$name] = 1;
|
||||
}
|
||||
|
||||
if ($name == 'modify' || $name == 'delete' or $name == 'create') {
|
||||
$mode = $name;
|
||||
|
||||
} else if ($name == 'node') {
|
||||
$curNodeAttrs = $attrs;
|
||||
$curNodeTags = array();
|
||||
|
||||
} else if ($name == 'tag') {
|
||||
$curNodeTags[$attrs['k']] = $attrs['v'];
|
||||
}
|
||||
}
|
||||
|
||||
function endElement ($parser, $name) {
|
||||
if (USE_STATISTICS) {
|
||||
global $mode, $deleteStmt, $deleteTagStmt, $deleteStatsStmt, $insertStmt, $insertTagStmt, $insertStatsStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $ts, $version, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate, $sql_statement;
|
||||
} else {
|
||||
global $mode, $deleteStmt, $deleteTagStmt, $insertStmt, $insertTagStmt, $selectStmt, $id, $latitude, $longitude, $k, $v, $curNodeAttrs, $curNodeTags, $mysqli, $countDelete, $countModifyDelete, $countModify, $countCreate, $sql_statement;
|
||||
}
|
||||
|
||||
if ($name == 'modify' || $name == 'delete' or $name == 'create') {
|
||||
$mode = '';
|
||||
} else if ($name == 'node') {
|
||||
if ($mode == 'delete') {
|
||||
$id = $curNodeAttrs['id'];
|
||||
|
||||
if (USE_STATISTICS) {
|
||||
if (! $deleteStatsStmt->execute()) {
|
||||
echo "***** Error : Deleting statistics $id : ". $deleteStatsStmt->error . "\n";
|
||||
}
|
||||
}
|
||||
|
||||
if (! $deleteTagStmt->execute()) {
|
||||
echo "***** Error : Deleting tags $id : ". $deleteTagStmt->error . "\n";
|
||||
}
|
||||
|
||||
if (! $deleteStmt->execute()) {
|
||||
echo "***** Error : Deleting $id : ". $deleteStmt->error . "\n";
|
||||
}
|
||||
if ($deleteStmt->affected_rows > 0) {
|
||||
$sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
|
||||
$sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
|
||||
$countDelete++;
|
||||
$mysqli->commit();
|
||||
printDebugCurNode();
|
||||
}
|
||||
|
||||
} else if ($mode == 'modify' || $mode == 'create') {
|
||||
if (! empty($curNodeTags)
|
||||
&& array_key_exists('man_made', $curNodeTags)
|
||||
&& $curNodeTags['man_made'] == 'surveillance') {
|
||||
|
||||
printDebugCurNode();
|
||||
|
||||
$id = $curNodeAttrs['id'];
|
||||
|
||||
if ($mode == 'modify') {
|
||||
$countModify++;
|
||||
|
||||
if (USE_STATISTICS) {
|
||||
if (! $deleteStatsStmt->execute()) {
|
||||
echo "***** Error : Deleting statistics $id for modification : ". $deleteStatsStmt->error . "\n";
|
||||
}
|
||||
}
|
||||
|
||||
if (! $deleteTagStmt->execute()) {
|
||||
echo "***** Error : Deleting tags $id for modification : ". $deleteTagStmt->error . "\n";
|
||||
}
|
||||
if (! $deleteStmt->execute()) {
|
||||
echo "***** Error : Deleting $id for modification : ". $deleteStmt->error . "\n";
|
||||
}
|
||||
} else {
|
||||
$countCreate++;
|
||||
}
|
||||
|
||||
$latitude = (int) ($curNodeAttrs['lat'] * 10000000);
|
||||
$longitude = (int) ($curNodeAttrs['lon'] * 10000000);
|
||||
$sql_statement = $sql_statement . "INSERT INTO position (id, latitude, longitude) VALUES ($id, $latitude , $longitude);\n";
|
||||
if (! $insertStmt->execute()) {
|
||||
echo "***** Error : inserting $id ($latitude x $longitude) : ". $insertStmt->error . "\n";
|
||||
}
|
||||
|
||||
$k = 'lat';
|
||||
$v = $curNodeAttrs['lat'];
|
||||
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||
if (! $insertTagStmt->execute()) {
|
||||
echo "***** Error : inserting latitude $v for $id : ". $insertTagStmt->error . "\n";
|
||||
}
|
||||
|
||||
$k = 'lon';
|
||||
$v = $curNodeAttrs['lon'];
|
||||
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||
if (! $insertTagStmt->execute()) {
|
||||
echo "***** Error : inserting longitude $v for $id : ". $insertTagStmt->error . "\n";
|
||||
}
|
||||
|
||||
$k = 'userid';
|
||||
$v = $curNodeAttrs['user'];
|
||||
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||
|
||||
if (! $insertTagStmt->execute()) {
|
||||
echo "***** Error : inserting user $v for $id : ". $insertTagStmt->error . "\n";
|
||||
}
|
||||
|
||||
$k = 'version';
|
||||
$v = $curNodeAttrs['version'];
|
||||
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||
if (! $insertTagStmt->execute()) {
|
||||
echo "***** Error : inserting version $v for $id : ". $insertTagStmt->error . "\n";
|
||||
}
|
||||
|
||||
$k = 'timestamp';
|
||||
$v = $curNodeAttrs['timestamp'];
|
||||
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||
if (! $insertTagStmt->execute()) {
|
||||
echo "***** Error : inserting timestamp $v for $id : ". $insertTagStmt->error . "\n";
|
||||
}
|
||||
|
||||
foreach($curNodeTags as $k => $v) {
|
||||
$sql_statement = $sql_statement . "INSERT INTO tag (id, k, v) VALUES ($id, '$k', '$v');\n";
|
||||
if (! $insertTagStmt->execute()) {
|
||||
echo "***** Error : inserting tag $k => $v for $id : ". $insertTagStmt->error . "\n";
|
||||
}
|
||||
}
|
||||
|
||||
if (USE_STATISTICS) {
|
||||
$ts = $curNodeAttrs['timestamp'];
|
||||
$version = $curNodeAttrs['version'];
|
||||
if (! $insertStatsStmt->execute()) {
|
||||
echo "***** Error : inserting ts $ts, version $version for $id : ". $insertStatsStmt->error . "\n";
|
||||
}
|
||||
}
|
||||
|
||||
$mysqli->commit();
|
||||
} else if ($mode == 'modify') {
|
||||
// delete former surveillance nodes that were modified to non-surveillance nodes
|
||||
$id = $curNodeAttrs['id'];
|
||||
|
||||
if (USE_STATISTICS) {
|
||||
if (! $deleteStatsStmt->execute()) {
|
||||
echo "***** Error : Deleting statistics $id for non-surveillance node : ". $deleteStatsStmt->error . "\n";
|
||||
}
|
||||
}
|
||||
if (! $deleteTagStmt->execute()) {
|
||||
echo "***** Error : Deleting tags $id for non-surveillance node : ". $deleteTagStmt->error . "\n";
|
||||
}
|
||||
if (! $deleteStmt->execute()) {
|
||||
echo "***** Error : Deleting $id for non-surveillance node : ". $deleteStmt->error . "\n";
|
||||
}
|
||||
|
||||
if ($deleteStmt->affected_rows > 0) {
|
||||
$sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
|
||||
$sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
|
||||
$countModifyDelete++;
|
||||
$mysqli->commit();
|
||||
printDebugCurNode();
|
||||
echo " ==> delete because modified to non-surveillance node\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$curNodeAttrs = null;
|
||||
$curNodeTags = null;
|
||||
}
|
||||
}
|
||||
|
||||
$file="change_file.osc";
|
||||
|
||||
$xml_parser = xml_parser_create();
|
||||
xml_parser_set_option($xml_parser, XML_OPTION_CASE_FOLDING, false);
|
||||
xml_set_element_handler($xml_parser, "startElement", "endElement");
|
||||
|
||||
if (!($fp = fopen($file, "r"))) {
|
||||
die("could not open XML input");
|
||||
}
|
||||
|
||||
while ($data = fread($fp, 4096)) {
|
||||
if (!xml_parse($xml_parser, $data, feof($fp))) {
|
||||
die(sprintf("XML error: %s at line %d",
|
||||
xml_error_string(xml_get_error_code($xml_parser)),
|
||||
xml_get_current_line_number($xml_parser)));
|
||||
}
|
||||
}
|
||||
|
||||
$sql="UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';";
|
||||
$mysqli->query($sql);
|
||||
$mysqli->commit();
|
||||
$sql_statement = $sql_statement . "UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';\n";
|
||||
$sql_statement = $sql_statement . "COMMIT;";
|
||||
$sql_file = "/migrations/".$_SERVER['argv'][1].".sql";
|
||||
|
||||
file_put_contents($sql_file, $sql_statement);
|
||||
printDebug();
|
||||
$mysqli->close();
|
||||
|
||||
?>
|
||||
93
data_handler/utils/create_camera_update_statements.sh
Normal file
93
data_handler/utils/create_camera_update_statements.sh
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
#!/bin/bash
|
||||
|
||||
exedir=$(cd `dirname "$0"`; pwd)
|
||||
|
||||
REPLICATE_URL=`grep "REPLICATE_URL" "$exedir/config.php" | sed -e 's/.*http/http/' -e 's#...$##' `
|
||||
|
||||
if [ -e "/var/lock/update_camera" ]
|
||||
then
|
||||
# Maybe an other update is running
|
||||
otherPid=$(cat "/var/lock/update_camera")
|
||||
count=$(ps $otherPid | grep -c `basename "$0"`)
|
||||
|
||||
if [ $count -gt 0 ]
|
||||
then
|
||||
echo "$0 is running yet. Exiting." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# OK. We can update the database...
|
||||
echo $$ > "/var/lock/update_camera"
|
||||
|
||||
exeDir=$(cd `dirname "$0"`; pwd);
|
||||
cd "$exeDir"
|
||||
|
||||
if [ -e "state.txt" ]
|
||||
then
|
||||
rm "state.txt"
|
||||
fi
|
||||
|
||||
#TODO get state from db
|
||||
# Read the last update timestamp
|
||||
# lastTimestamp=$(grep "^timestamp=" "lastState.txt" | cut -d'=' -f2-)
|
||||
# lastSeqNum=$(grep "^sequenceNumber=" "lastState.txt" | cut -d'=' -f2-)
|
||||
|
||||
lastSeqNum=$(php get_sync_state.php)
|
||||
|
||||
curl -L "$REPLICATE_URL/state.txt" -o state.txt
|
||||
|
||||
newTimestamp=$(grep "^timestamp=" "state.txt" | cut -d'=' -f2-)
|
||||
newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-)
|
||||
|
||||
if [ $newSeqNum -eq $lastSeqNum ]
|
||||
then
|
||||
echo "No new file to be processed"
|
||||
exit 1
|
||||
fi
|
||||
mkdir logs
|
||||
curSeqNum=$lastSeqNum
|
||||
while [ $curSeqNum -lt $newSeqNum ]
|
||||
do
|
||||
curSeqNum=$(( $curSeqNum + 1 ))
|
||||
|
||||
logFileName="logs/log.$curSeqNum"
|
||||
|
||||
if [ -e "change_file.osc" ]
|
||||
then
|
||||
rm "change_file.osc"
|
||||
fi
|
||||
|
||||
if [ -e "change_file.osc.gz" ]
|
||||
then
|
||||
rm "change_file.osc.gz"
|
||||
fi
|
||||
|
||||
echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName"
|
||||
|
||||
targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'`
|
||||
targetDirName="$REPLICATE_URL/$targetDirName"
|
||||
|
||||
curl -L "$targetDirName" -o change_file.osc.gz 2>&1 | tee -a $logFileName
|
||||
if [ $? -gt 0 ]
|
||||
then
|
||||
echo "Error during recovery of $targetDirName" | tee -a $logFileName
|
||||
exit 1
|
||||
fi
|
||||
|
||||
gunzip change_file.osc.gz 2>&1 | tee -a $logFileName
|
||||
if [ $? -gt 0 ]
|
||||
then
|
||||
echo "Error during decompression of $targetDirName" | tee -a $logFileName
|
||||
exit 1
|
||||
fi
|
||||
|
||||
php create_camera_update_statements.php $curSeqNum 2>&1 | tee -a $logFileName
|
||||
|
||||
targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'`
|
||||
|
||||
curl -L "$targetDirName" -o lastState.txt 2>&1 | tee -a $logFileName
|
||||
echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName"
|
||||
done
|
||||
|
||||
rm "/var/lock/update_camera"
|
||||
Loading…
Add table
Add a link
Reference in a new issue