improved performance

data_handler/utils/create_camera_update_statements_download.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+exeDir=$(cd "$(dirname "$0")"; pwd)
+cd "$exeDir"
+
+REPLICATE_URL=$(grep -oP "REPLICATE_URL[',\s]*['\"]\Khttps?://[^'\"]+" config.php)
+
+echo "Using replicate URL: $REPLICATE_URL"
+export REPLICATE_URL
+
+
+lastSeqNum=$(php get_sync_state.php)
+curl -sL "$REPLICATE_URL/state.txt" -o state.txt
+newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-)
+echo $newSeqNum
+if [ "$newSeqNum" -eq "$lastSeqNum" ]; then
+  echo "No new file to be processed"
+  exit 0
+fi
+
+mkdir -p logs
+seq "$((lastSeqNum + 1))" "$newSeqNum" > seq_list.txt
+
+
+cat seq_list.txt | xargs -n1 -P7 -I{} bash -c '
+  seqnum="$1"
+  f=$(printf "%09d" "$seqnum")
+  path=$(echo "$f" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
+  url="${REPLICATE_URL}/${path}.osc.gz"
+  echo "Downloading $url"
+  curl -sL "$url" -o "${seqnum}.osc.gz" && gunzip "${seqnum}.osc.gz"
+' _ {}