# .gitlab-ci.yml
#
# cumulus
#
# pipelines can be triggered manually in the web
stages:
- test
- build
- benchmarks-build
- benchmarks-run
- publish
default:
interruptible: true
retry:
max: 2
when:
- runner_system_failure
- unknown_failure
- api_failure
variables:
GIT_STRATEGY: fetch
GIT_DEPTH: 100
CARGO_INCREMENTAL: 0
CI_IMAGE: "paritytech/ci-linux:production"
DOCKER_OS: "debian:stretch"
ARCH: "x86_64"
.collect-artifacts: &collect-artifacts
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
when: on_success
expire_in: 1 days
paths:
- ./artifacts/
.rust-info-script: &rust-info-script
- rustup show
- cargo --version
- rustup +nightly show
- cargo +nightly --version
- bash --version
.common-refs: &common-refs
# these jobs run always*
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000
.publish-refs: &publish-refs
rules:
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
# run benchmarks manually only on release-parachains-v* branch
.benchmarks-manual-refs: &benchmarks-manual-refs
rules:
- if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000
when: manual
# run benchmarks only on release-parachains-v* branch
.benchmarks-refs: &benchmarks-refs
rules:
- if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000
.docker-env: &docker-env
image: "${CI_IMAGE}"
before_script:
- *rust-info-script
tags:
- linux-docker
.kubernetes-env: &kubernetes-env
image: "${CI_IMAGE}"
tags:
- kubernetes-parity-build
#### stage: test
test-linux-stable:
stage: test
<<: *docker-env
<<: *common-refs
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
script:
- time cargo test --all --release --locked -- --include-ignored
check-runtime-benchmarks:
stage: test
<<: *docker-env
<<: *common-refs
script:
# Check that the node will compile with `runtime-benchmarks` feature flag.
- time cargo check --all --features runtime-benchmarks
# Check that parachain-template will compile with `runtime-benchmarks` feature flag.
- time cargo check -p parachain-template-node --features runtime-benchmarks
cargo-check-try-runtime:
stage: test
<<: *docker-env
<<: *common-refs
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
needs:
- job: check-runtime-benchmarks
artifacts: false
script:
# Check that the node will compile with `try-runtime` feature flag.
- time cargo check --all --features try-runtime
# Check that parachain-template will compile with `try-runtime` feature flag.
- time cargo check -p parachain-template-node --features try-runtime
check-rustdoc:
stage: test
<<: *docker-env
<<: *common-refs
variables:
SKIP_WASM_BUILD: 1
RUSTDOCFLAGS: "-Dwarnings"
script:
- time cargo +nightly doc --workspace --all-features --verbose --no-deps
cargo-check-benches:
stage: test
<<: *docker-env
<<: *common-refs
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
needs:
- job: check-rustdoc
artifacts: false
script:
- time cargo check --all --benches
#### stage: build
build-linux-stable:
stage: build
<<: *docker-env
<<: *collect-artifacts
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
needs:
- job: check-rustdoc
artifacts: false
script:
- echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___"
- time cargo build --release --locked --bin polkadot-parachain
- echo "___Packing the artifacts___"
- mkdir -p ./artifacts
- mv ./target/release/polkadot-parachain ./artifacts/.
- echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___"
- echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION
build-test-parachain:
stage: build
<<: *docker-env
<<: *collect-artifacts
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
needs:
- job: check-rustdoc
artifacts: false
script:
- echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___"
- time cargo +nightly build --release --locked --bin test-parachain
- echo "___Packing the artifacts___"
- mkdir -p ./artifacts
- mv ./target/release/test-parachain ./artifacts/.
#### stage: publish
.build-push-image: &build-push-image
image: quay.io/buildah/stable
variables:
DOCKERFILE: "" # docker/path-to.Dockerfile
IMAGE_NAME: "" # docker.io/paritypr/image_name
VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
script:
- test "$PARITYPR_USER" -a "$PARITYPR_PASS" ||
( echo "no docker credentials provided"; exit 1 )
- buildah bud
--format=docker
--build-arg VCS_REF="${CI_COMMIT_SHA}"
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
--build-arg IMAGE_NAME="${IMAGE_NAME}"
--tag "$IMAGE_NAME:$VERSION"
--file ${DOCKERFILE} .
- echo "$PARITYPR_PASS" |
buildah login --username "$PARITYPR_USER" --password-stdin docker.io
- buildah info
- buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
after_script:
- buildah logout --all
build-push-image-polkadot-parachain-debug:
stage: publish
<<: *kubernetes-env
<<: *common-refs
<<: *build-push-image
needs:
- job: build-linux-stable
artifacts: true
variables:
DOCKERFILE: "docker/polkadot-parachain-debug_unsigned_injected.Dockerfile"
IMAGE_NAME: "docker.io/paritypr/polkadot-parachain-debug"
VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
build-push-image-test-parachain:
stage: publish
<<: *kubernetes-env
<<: *common-refs
<<: *build-push-image
needs:
- job: build-test-parachain
artifacts: true
variables:
DOCKERFILE: "docker/test-parachain_injected.Dockerfile"
IMAGE_NAME: "docker.io/paritypr/test-parachain"
VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
publish-s3:
stage: publish
<<: *kubernetes-env
image: paritytech/awscli:latest
<<: *publish-refs
needs:
- job: build-linux-stable
artifacts: true
variables:
GIT_STRATEGY: none
BUCKET: "releases.parity.io"
PREFIX: "cumulus/${ARCH}-${DOCKER_OS}"
script:
- echo "___Publishing a binary with debug assertions!___"
- echo "___VERSION = $(cat ./artifacts/VERSION) ___"
- aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/
- echo "___Updating objects in latest path___"
- aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
after_script:
- aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
--recursive --human-readable --summarize
#### stage: benchmarks
# Work only on release-parachains-v* branches
benchmarks-build:
stage: benchmarks-build
<<: *docker-env
<<: *collect-artifacts
<<: *benchmarks-manual-refs
script:
- time cargo build --profile production --locked --features runtime-benchmarks
- mkdir -p artifacts
- cp target/production/polkadot-parachain ./artifacts/
.git-commit-push: &git-commit-push
- git status
# Set git config
- rm -rf .git/config
- git config --global user.email "${GITHUB_EMAIL}"
- git config --global user.name "${GITHUB_USER}"
- git config remote.origin.url "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/paritytech/${CI_PROJECT_NAME}.git"
- git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
# push results to github
- git checkout -b $BRANCHNAME
- git add parachains/*
- git commit -m "[benchmarks] pr with weights"
- git push origin $BRANCHNAME
benchmarks-statemint:
stage: benchmarks-run
before_script:
- *rust-info-script
<<: *collect-artifacts
<<: *benchmarks-refs
script:
- ./scripts/benchmarks-ci.sh assets statemine ./artifacts
- ./scripts/benchmarks-ci.sh assets statemint ./artifacts
- ./scripts/benchmarks-ci.sh assets westmint ./artifacts
- export BRANCHNAME="weights-statemint-${CI_COMMIT_BRANCH}"
- *git-commit-push
# create PR
- curl -u ${GITHUB_USER}:${GITHUB_TOKEN}
-d '{"title":"[benchmarks] Update weights for statemine/t","body":"This PR is generated automatically by CI.","head":"'$BRANCHNAME'","base":"'${CI_COMMIT_BRANCH}'"}'
-X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls
after_script:
- rm -rf .git/config
tags:
- weights
benchmarks-collectives:
stage: benchmarks-run
before_script:
- *rust-info-script
<<: *collect-artifacts
<<: *benchmarks-refs
script:
- ./scripts/benchmarks-ci.sh collectives collectives-polkadot ./artifacts
- git status
- export BRANCHNAME="weights-collectives-${CI_COMMIT_BRANCH}"
- *git-commit-push
# create PR
- curl -u ${GITHUB_USER}:${GITHUB_TOKEN}
-d '{"title":"[benchmarks] Update weights for collectives","body":"This PR is generated automatically by CI.","head":"'$BRANCHNAME'","base":"'${CI_COMMIT_BRANCH}'"}'
-X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls
after_script:
- rm -rf .git/config
tags:
- weights
publish-benchmarks-statemint-s3: &publish-benchmarks
stage: publish
<<: *kubernetes-env
image: paritytech/awscli:latest
<<: *benchmarks-refs
needs:
- job: benchmarks-statemint
artifacts: true
variables:
GIT_STRATEGY: none
BUCKET: "releases.parity.io"
PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks"
script:
- echo "___Removing binary from artifacts___"
- rm -f ./artifacts/polkadot-parachain
- echo "___Publishing benchmark results___"
- aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/
after_script:
- aws s3 ls s3://${BUCKET}/${PREFIX}/ --recursive --human-readable --summarize
publish-benchmarks-collectives-s3:
<<: *publish-benchmarks
needs:
- job: benchmarks-collectives
artifacts: true
#### stage: .post
# This job cancels the whole pipeline if any of provided jobs fail.
# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
# to fail the pipeline as soon as possible to shorten the feedback loop.
cancel-pipeline:
stage: .post
needs:
- job: test-linux-stable
artifacts: false
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
when: on_failure
variables:
PROJECT_ID: "${CI_PROJECT_ID}"
PIPELINE_ID: "${CI_PIPELINE_ID}"
trigger: "parity/infrastructure/ci_cd/pipeline-stopper"