increased speed to create_sql

data_handler/utils/create_camera_update_statements.php

@@ -23,7 +23,7 @@ if($mysqli->connect_errno) {
   echo "Error while connecting to DB : $mysqli->error \n" ;
   exit(1);
 }
-$mysqli->autocommit(FALSE);
+
 
 if (! ($deleteStmt = $mysqli->prepare("DELETE FROM position WHERE id=?"))) {
   echo "Error while preparing delete position statement : " . $mysqli->error ;
@@ -153,9 +153,6 @@ function endElement ($parser, $name) {
       if ($deleteStmt->affected_rows > 0) {
         $sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
         $sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
-        $countDelete++;
-        $mysqli->commit();
-        printDebugCurNode();
       }
 
     } else if ($mode == 'modify' || $mode == 'create') {
@@ -163,12 +160,11 @@ function endElement ($parser, $name) {
           && array_key_exists('man_made', $curNodeTags)
           && $curNodeTags['man_made'] == 'surveillance') {
 
-        printDebugCurNode();
+        
 
         $id = $curNodeAttrs['id'];
 
         if ($mode == 'modify') {
-          $countModify++;
 
           if (USE_STATISTICS) {
             if (! $deleteStatsStmt->execute()) {
@@ -183,7 +179,6 @@ function endElement ($parser, $name) {
             echo "***** Error : Deleting $id for modification : ". $deleteStmt->error . "\n";
           }
         } else {
-          $countCreate++;
         }
 
         $latitude = (int) ($curNodeAttrs['lat'] * 10000000);
@@ -264,10 +259,6 @@ function endElement ($parser, $name) {
         if ($deleteStmt->affected_rows > 0) {
           $sql_statement = $sql_statement . "DELETE FROM position WHERE id=$id;\n";
           $sql_statement = $sql_statement . "DELETE FROM tag WHERE id=$id;\n";
-          $countModifyDelete++;
-          $mysqli->commit();
-          printDebugCurNode();
-          echo "  ==> delete because modified to non-surveillance node\n";
         }
       }
     }
@@ -297,13 +288,11 @@ while ($data = fread($fp, 4096)) {
 
 $sql="UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';";
 $mysqli->query($sql);
-$mysqli->commit();
 $sql_statement = $sql_statement . "UPDATE sync_state SET v='" . $_SERVER['argv'][1] ."' WHERE k='sequenceNumber';\n";
 $sql_statement = $sql_statement . "COMMIT;";
 $sql_file = "/migrations/".$_SERVER['argv'][1].".sql";
 
 file_put_contents($sql_file, $sql_statement);
-printDebug();
 $mysqli->close();
 
 ?>

data_handler/utils/create_camera_update_statements.sh

@@ -51,43 +51,17 @@ while [ $curSeqNum -lt $newSeqNum ]
 do
   curSeqNum=$(( $curSeqNum + 1 ))
 
-  logFileName="logs/log.$curSeqNum"
+  rm "change_file.osc"
 
-  if [ -e "change_file.osc" ]
-  then
-    rm "change_file.osc"
-  fi
-
-  if [ -e "change_file.osc.gz" ]
-  then
-    rm "change_file.osc.gz"
-  fi
-
-  echo `date "+%d/%m/%Y %H:%M"`" - Start processing sequence number $curSeqNum" > "$logFileName"
 
   targetDirName=`echo "000000000$curSeqNum" | sed 's#.*\(...\)\(...\)\(...\)$#\1/\2/\3.osc.gz#'`
   targetDirName="$REPLICATE_URL/$targetDirName"
 
-  curl -L "$targetDirName" -o change_file.osc.gz 2>&1 | tee -a $logFileName
-  if [ $? -gt 0 ]
-  then
-    echo "Error during recovery of $targetDirName" | tee -a $logFileName
-    exit 1
-  fi
+  curl -L "$targetDirName" -o change_file.osc.gz
 
-  gunzip change_file.osc.gz 2>&1 | tee -a $logFileName
-  if [ $? -gt 0 ]
-  then
-    echo "Error during decompression of $targetDirName" | tee -a $logFileName
-    exit 1
-  fi
+  gunzip change_file.osc.gz 2>&1
 
-  php create_camera_update_statements.php $curSeqNum 2>&1 | tee -a $logFileName
-
-  targetDirName=`echo "$targetDirName" | sed 's/.osc.gz$/.state.txt/'`
-
-  curl -L "$targetDirName" -o lastState.txt 2>&1 | tee -a $logFileName
-  echo `date "+%d/%m/%Y %H:%M"`" - Finish processing sequence number $curSeqNum" | tee -a "$logFileName"
+  php create_camera_update_statements.php $curSeqNum 2>&1
 done
 
 rm "/var/lock/update_camera"

get_rate.sh

@@ -0,0 +1,5 @@
+before=$(ls migrations | wc -l)
+sleep 60
+after=$(ls migrations | wc -l)
+let dif=after-before
+echo "$dif"