CCCHH/sunders
CCCHH/sunders
3
0
Fork 1
forked from kamba4/sunders
Code Issues 5 Pull requests 3 Packages 1 Activity Actions

moved persistence
All checks were successful
Build (and tag) Images / build (push) Successful in 37s
Details

Browse source
This commit is contained in:
aso 2025-10-19 18:44:52 +02:00
commit 4c28ff9b0a
3 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data_handler/utils/create_camera_update_statements_download.sh
View file

@ -28,5 +28,5 @@ cat seq_list.txt | xargs -n1 -P10 -I{} bash -c '
path=$(echo "$f" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
url="${REPLICATE_URL}/${path}.osc.gz"
echo "Downloading $url"
curl -sL "$url" -o "${seqnum}.osc.gz" && gunzip "${seqnum}.osc.gz" && python filter_osc.py ${seqnum} && rm ${seqnum}.osc
curl -sL "$url" -o /migrations/"${seqnum}.osc.gz" && gunzip /migrations/"${seqnum}.osc.gz" && python filter_osc.py ${seqnum} && rm /migrations/${seqnum}.osc
' _ {}

8
data_handler/utils/create_camera_update_statements_process.sh
View file

@ -50,21 +50,21 @@ curSeqNum=$lastSeqNum
curSeqNum=$(( $curSeqNum + 1 ))
while [ $curSeqNum -lt $newSeqNum ]
do
if [ -e "${curSeqNum}_filtered.osc" ]
if [ -e /migrations/"${curSeqNum}_filtered.osc" ]
then
rm "change_file.osc"
cp ${curSeqNum}_filtered.osc change_file.osc
cp /migrations/${curSeqNum}_filtered.osc change_file.osc
php create_camera_update_statements.php $curSeqNum
rm ${curSeqNum}_filtered.osc
rm /migrations/${curSeqNum}_filtered.osc
curSeqNum=$(( $curSeqNum + 1 ))
else
path=$(echo "$(printf "%09d" "$curSeqNum")" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
url="${REPLICATE_URL}/${path}.osc.gz"
echo $url
curl -sL "$url" -o "$curSeqNum.osc.gz" && gunzip "$curSeqNum.osc.gz" && python filter_osc.py ${curSeqNum} && rm ${curSeqNum}.osc
curl -sL "$url" -o /migrations/"$curSeqNum.osc.gz" && gunzip /migrations/"$curSeqNum.osc.gz" && python filter_osc.py ${curSeqNum} && rm /migrations/${curSeqNum}.osc
fi
done

4
data_handler/utils/filter_osc.py
View file

@ -9,8 +9,8 @@ def open_maybe_gzip(filename, mode="rb"):
return open(filename, mode)
def process_file(number):
input_file = f"{number}.osc"
filtered_file = f"{number}_filtered.osc"
input_file = f"/migrations/{number}.osc"
filtered_file = f"/migrations/{number}_filtered.osc"
deleted_file = f"/migrations/{number}_deleted.osc"
# --- open input and outputs ---