CCCHH/sunders
CCCHH/sunders
3
0
Fork 1
forked from kamba4/sunders
Code Issues 5 Pull requests 3 Packages 1 Activity Actions

moved persistence
All checks were successful
Build (and tag) Images / build (push) Successful in 37s
Details

Browse source
This commit is contained in:
aso 2025-10-19 18:44:52 +02:00
commit 4c28ff9b0a
3 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data_handler/utils/create_camera_update_statements_download.sh
View file

@ -28,5 +28,5 @@ cat seq_list.txt | xargs -n1 -P10 -I{} bash -c '
path=$(echo "$f" | sed -E "s#(...)(...)(...)#\1/\2/\3#") path=$(echo "$f" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
url="${REPLICATE_URL}/${path}.osc.gz" url="${REPLICATE_URL}/${path}.osc.gz"
echo "Downloading $url" echo "Downloading $url"
curl -sL "$url" -o "${seqnum}.osc.gz" && gunzip "${seqnum}.osc.gz" && python filter_osc.py ${seqnum} && rm ${seqnum}.osc curl -sL "$url" -o /migrations/"${seqnum}.osc.gz" && gunzip /migrations/"${seqnum}.osc.gz" && python filter_osc.py ${seqnum} && rm /migrations/${seqnum}.osc
' _ {} ' _ {}

8
data_handler/utils/create_camera_update_statements_process.sh
View file

@ -50,21 +50,21 @@ curSeqNum=$lastSeqNum
curSeqNum=$(( $curSeqNum + 1 )) curSeqNum=$(( $curSeqNum + 1 ))
while [ $curSeqNum -lt $newSeqNum ] while [ $curSeqNum -lt $newSeqNum ]
do do
if [ -e "${curSeqNum}_filtered.osc" ] if [ -e /migrations/"${curSeqNum}_filtered.osc" ]
then then
rm "change_file.osc" rm "change_file.osc"
cp ${curSeqNum}_filtered.osc change_file.osc cp /migrations/${curSeqNum}_filtered.osc change_file.osc
php create_camera_update_statements.php $curSeqNum php create_camera_update_statements.php $curSeqNum
rm ${curSeqNum}_filtered.osc rm /migrations/${curSeqNum}_filtered.osc
curSeqNum=$(( $curSeqNum + 1 )) curSeqNum=$(( $curSeqNum + 1 ))
else else
path=$(echo "$(printf "%09d" "$curSeqNum")" | sed -E "s#(...)(...)(...)#\1/\2/\3#") path=$(echo "$(printf "%09d" "$curSeqNum")" | sed -E "s#(...)(...)(...)#\1/\2/\3#")
url="${REPLICATE_URL}/${path}.osc.gz" url="${REPLICATE_URL}/${path}.osc.gz"
echo $url echo $url
curl -sL "$url" -o "$curSeqNum.osc.gz" && gunzip "$curSeqNum.osc.gz" && python filter_osc.py ${curSeqNum} && rm ${curSeqNum}.osc curl -sL "$url" -o /migrations/"$curSeqNum.osc.gz" && gunzip /migrations/"$curSeqNum.osc.gz" && python filter_osc.py ${curSeqNum} && rm /migrations/${curSeqNum}.osc
fi fi
done done

4
data_handler/utils/filter_osc.py
View file

@ -9,8 +9,8 @@ def open_maybe_gzip(filename, mode="rb"):
return open(filename, mode) return open(filename, mode)
def process_file(number): def process_file(number):
input_file = f"{number}.osc" input_file = f"/migrations/{number}.osc"
filtered_file = f"{number}_filtered.osc" filtered_file = f"/migrations/{number}_filtered.osc"
deleted_file = f"/migrations/{number}_deleted.osc" deleted_file = f"/migrations/{number}_deleted.osc"
# --- open input and outputs --- # --- open input and outputs ---