-
Notifications
You must be signed in to change notification settings - Fork 7
/
20_ingest-verify_test.sh
100 lines (83 loc) · 3.27 KB
/
20_ingest-verify_test.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
#!/bin/sh
set -e
# install tools if missing
if [ ! "$(command -v xxd)" ]; then
if [ "$(id -u)" != 0 ]; then
echo "xxd is missing, unable to install it"
exit 1
fi
apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
apt-get -o DPkg::Lock::Timeout=60 install -y xxd >/dev/null
fi
cd shared || true
token="$(curl http://oidc:8080/tokens | jq -r '.[0]')"
# we need to insert the key hash into the db before ingesting files
# decode from base64, then hex encode the public key
key="$(head -n2 /shared/sync.pub.pem | tail -n1 | base64 -d | xxd -p | tr -d '\n')"
for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
ENC_SHA=$(sha256sum "$file.c4gh" | cut -d' ' -f 1)
ENC_MD5=$(md5sum "$file.c4gh" | cut -d' ' -f 1)
## get correlation id from upload message
CORRID=$(
curl -s -X POST \
-H "content-type:application/json" \
-u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \
-d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id
)
## publish message to trigger ingestion
properties=$(
jq -c -n \
--argjson delivery_mode 2 \
--arg correlation_id "$CORRID" \
--arg content_encoding UTF-8 \
--arg content_type application/json \
'$ARGS.named'
)
encrypted_checksums=$(
jq -c -n \
--arg sha256 "$ENC_SHA" \
--arg md5 "$ENC_MD5" \
'$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))'
)
ingest_payload=$(
jq -r -c -n \
--arg type ingest \
--arg user test@dummy.org \
--arg filepath test_dummy.org/"$file.c4gh" \
--argjson encrypted_checksums "$encrypted_checksums" \
'$ARGS.named|@base64'
)
ingest_body=$(
jq -c -n \
--arg vhost sda \
--arg name sda \
--argjson properties "$properties" \
--arg routing_key "ingest" \
--arg payload_encoding base64 \
--arg payload "$ingest_payload" \
'$ARGS.named'
)
curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \
-H 'Content-Type: application/json;charset=UTF-8' \
-d "$ingest_body" | jq
# Insert key hash after the ingestion of the first file has started
# Makes sure that ingestion works even if the key hash is there
curl -k -L "http://api:8080/key/hashed" -H "Authorization: Bearer $token" -d "{\"hash\": \"$key\", \"description\": \"first key\"}"
done
echo "waiting for verify to complete"
RETRY_TIMES=0
until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/verified/ | jq -r '.messages_ready')" -eq 4 ]; do
echo "waiting for verify to complete"
RETRY_TIMES=$((RETRY_TIMES + 1))
if [ "$RETRY_TIMES" -eq 30 ]; then
echo "::error::Time out while waiting for verify to complete"
exit 1
fi
sleep 2
done
key_hashes="$(psql -U postgres -h postgres -d sda -At -c "select distinct key_hash from sda.files" | wc -l)"
if [ "$key_hashes" -lt 0 ]; then
echo "::error::Ingested files did not have any key hashes."
exit 1
fi
echo "ingestion and verification test completed successfully"