-
Notifications
You must be signed in to change notification settings - Fork 0
/
fetch.sh
30 lines (24 loc) · 1023 Bytes
/
fetch.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
FORMAT=parquet
flowmaps-data daily_mobility list | sed 1d |
while read -r line
do
SOURCE_LAYER=`echo "$line" | python3 -c "import json, sys; d = json.load(sys.stdin); print(d['source_layer'], end='');"`
TARGET_LAYER=`echo "$line" | python3 -c "import json, sys; d = json.load(sys.stdin); print(d['target_layer'], end='');"`
DATES=`flowmaps-data daily_mobility list-dates`
mkdir -p data/$SOURCE_LAYER-$TARGET_LAYER/
for date in $DATES; do
echo $date
FILENAME=data/$SOURCE_LAYER-$TARGET_LAYER/$date.$SOURCE_LAYER-$TARGET_LAYER.daily_mobility.$FORMAT
if [ -f "$FILENAME" ]; then
echo "skipping $FILENAME (already exists)"
continue
fi
flowmaps-data daily_mobility download \
--start-date $date \
--end-date $date \
--source-layer $SOURCE_LAYER \
--target-layer $TARGET_LAYER \
--output-format $FORMAT \
--output-file $FILENAME
done
done