Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.

Commit 3883f73

Browse files
TriplEightTriplEight
authored
CI: build a test artifact (#527)
* CI: build a test artifact * CI: try building before testing * CI: try proper bin name * CI: preempty cache to benchmark properly * CI: run build after test * CI: change target source * CI: try build before test once again * CI: chore * CI: prepopulate target cache * CI: conditional build; aws sync * CI: mock building to test publishing * CI: lacks permissions in the other job * CI: undebug
1 parent 6f47b85 commit 3883f73

File tree

2 files changed

+123
-25
lines changed

2 files changed

+123
-25
lines changed

‎.gitlab-ci.yml

+95-25
Original file line numberDiff line numberDiff line change
@@ -7,59 +7,129 @@
77

88
stages:
99
- test
10-
- build
10+
- publish
1111

1212
variables: &default-vars
1313
GIT_STRATEGY: fetch
14-
GIT_DEPTH: 3
14+
GIT_DEPTH: 100
1515
CARGO_INCREMENTAL: 0
1616
CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
17-
CI_SERVER_NAME: "GitLab CI"
17+
CI_IMAGE: "paritytech/ci-linux:production"
18+
DOCKER_OS: "debian:stretch"
19+
ARCH: "x86_64"
20+
21+
.rust-info-script: &rust-info-script
22+
- rustup show
23+
- cargo --version
24+
- rustup +nightly show
25+
- cargo +nightly --version
26+
- bash --version
27+
- sccache -s
28+
29+
.publish-refs: &publish-refs
30+
rules:
31+
- if: $CI_PIPELINE_SOURCE == "web"
32+
- if: $CI_PIPELINE_SOURCE == "schedule"
33+
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
1834

1935
.docker-env: &docker-env
20-
image: paritytech/ci-linux:production
36+
image: "${CI_IMAGE}"
2137
before_script:
22-
- cargo -vV
23-
- rustc -vV
24-
- rustup show
25-
- cargo --version
38+
- *rust-info-script
39+
- ./scripts/ci/pre_cache.sh
2640
- sccache -s
27-
only:
28-
- master
29-
- /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
30-
- schedules
31-
- web
32-
- /^[0-9]+$/ # PRs
33-
dependencies: []
34-
interruptible: true
3541
retry:
3642
max: 2
3743
when:
3844
- runner_system_failure
3945
- unknown_failure
4046
- api_failure
47+
interruptible: true
4148
tags:
4249
- linux-docker
4350

44-
#### stage: test
51+
.kubernetes-env: &kubernetes-env
52+
retry:
53+
max: 2
54+
when:
55+
- runner_system_failure
56+
- unknown_failure
57+
- api_failure
58+
interruptible: true
59+
tags:
60+
- kubernetes-parity-build
4561

46-
cargo-audit:
47-
stage: test
48-
<<: *docker-env
49-
except:
50-
- /^[0-9]+$/
51-
script:
52-
- cargo audit
53-
allow_failure: true
62+
.collect-artifacts: &collect-artifacts
63+
artifacts:
64+
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
65+
when: on_success
66+
expire_in: 28 days
67+
paths:
68+
- ./artifacts/
69+
70+
#### stage: test
5471

5572
test-linux-stable:
5673
stage: test
5774
<<: *docker-env
75+
rules:
76+
- if: $CI_COMMIT_REF_NAME == "master"
77+
- if: $CI_COMMIT_REF_NAME == "tags"
78+
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
79+
# It doesn't make sense to build on every commit, so we build on tags
80+
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
81+
variables:
82+
ARE_WE_RELEASING_YET: maybe!
83+
# web and schedule triggers can be provided with the non-empty variable ARE_WE_RELEASING_YET
84+
# to run building and publishing the binary.
85+
- if: $CI_PIPELINE_SOURCE == "web"
86+
- if: $CI_PIPELINE_SOURCE == "schedule"
87+
<<: *collect-artifacts
5888
variables:
5989
<<: *default-vars
6090
# Enable debug assertions since we are running optimized builds for testing
6191
# but still want to have debug assertions.
6292
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
6393
script:
6494
- time cargo test --all --release --locked
95+
# It's almost free to produce a binary here, please refrain from using it in production since
96+
# it goes with the debug assertions.
97+
- if [ "${ARE_WE_RELEASING_YET}" ]; then
98+
echo "___Building a binary___";
99+
time cargo build --release --locked --bin polkadot-collator;
100+
echo "___Packing the artifacts___";
101+
mkdir -p ./artifacts;
102+
mv ${CARGO_TARGET_DIR}/release/polkadot-collator ./artifacts/.;
103+
echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___";
104+
echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION;
105+
else
106+
exit 0;
107+
fi
65108
- sccache -s
109+
110+
#### stage: publish
111+
112+
publish-s3:
113+
stage: publish
114+
<<: *kubernetes-env
115+
image: paritytech/awscli:latest
116+
<<: *publish-refs
117+
variables:
118+
GIT_STRATEGY: none
119+
BUCKET: "releases.parity.io"
120+
PREFIX: "cumulus/${ARCH}-${DOCKER_OS}"
121+
before_script:
122+
# Job will fail if no artifacts were provided by test-linux-stable job. It's only possible for
123+
# this test to fail if the pipeline was triggered by web or schedule trigger without supplying
124+
# a nono-empty ARE_WE_RELEASING_YET variable.
125+
- test -e ./artifacts/polkadot-collator ||
126+
( echo "___No artifacts were provided by the previous job, please check the build there___"; exit 1 )
127+
script:
128+
- echo "___Publishing a binary with debug assertions!___"
129+
- echo "___VERSION = $(cat ./artifacts/VERSION) ___"
130+
- aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/
131+
- echo "___Updating objects in latest path___"
132+
- aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
133+
after_script:
134+
- aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
135+
--recursive --human-readable --summarize

‎scripts/ci/pre_cache.sh

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
set -u
4+
5+
# if there is no directory for this $CI_COMMIT_REF_NAME/$CI_JOB_NAME
6+
# create such directory and
7+
# copy recursively all the files from the newest dir which has $CI_JOB_NAME, if it exists
8+
9+
# cache lives in /ci-cache/${CI_PROJECT_NAME}/${2}/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}
10+
11+
function prepopulate {
12+
if [[ ! -d $1 ]]; then
13+
mkdir -p "/ci-cache/$CI_PROJECT_NAME/$2/$CI_COMMIT_REF_NAME";
14+
FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \
15+
-type d -name "$CI_JOB_NAME" -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2);
16+
if [[ -d $FRESH_CACHE ]]; then
17+
echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____";
18+
time cp -r "$FRESH_CACHE" "$1";
19+
else
20+
echo "_____No such $2 dir, proceeding from scratch_____";
21+
fi
22+
else
23+
echo "____No need to prepopulate $2 cache____";
24+
fi
25+
}
26+
27+
# CARGO_HOME cache is still broken so would be handled some other way.
28+
prepopulate "$CARGO_TARGET_DIR" targets

0 commit comments

Comments
 (0)