forked from kamba4/sunders
32 lines
931 B
Bash
32 lines
931 B
Bash
#!/bin/bash
|
|
|
|
exeDir=$(cd "$(dirname "$0")"; pwd)
|
|
cd "$exeDir"
|
|
|
|
REPLICATE_URL=$(grep -oP "REPLICATE_URL[',\s]*['\"]\Khttps?://[^'\"]+" config.php)
|
|
|
|
echo "Using replicate URL: $REPLICATE_URL"
|
|
export REPLICATE_URL
|
|
|
|
|
|
lastSeqNum=$(php get_sync_state.php)
|
|
curl -sL "$REPLICATE_URL/state.txt" -o state.txt
|
|
newSeqNum=$(grep "^sequenceNumber=" "state.txt" | cut -d'=' -f2-)
|
|
echo $newSeqNum
|
|
if [ "$newSeqNum" -eq "$lastSeqNum" ]; then
|
|
echo "No new file to be processed"
|
|
exit 0
|
|
fi
|
|
|
|
mkdir -p logs
|
|
seq "$((lastSeqNum + 1))" "$newSeqNum" > seq_list.txt
|
|
|
|
|
|
cat seq_list.txt | xargs -n1 -P10 -I{} bash -c '
|
|
seqnum="$1"
|
|
f=$(printf "%09d" "$seqnum")
|
|
path=$(echo "$f" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
|
|
url="${REPLICATE_URL}/${path}.osc.gz"
|
|
echo "Downloading $url"
|
|
curl -sL "$url" -o /migrations/"${seqnum}.osc.gz" && gunzip /migrations/"${seqnum}.osc.gz" && python filter_osc.py ${seqnum} && rm /migrations/${seqnum}.osc
|
|
' _ {}
|