Merge branch 'master' into snowbridge

# Conflicts: # .github/workflows/check-labels.yml # .github/workflows/check-prdoc.yml # .github/workflows/check-workspace.yml # .github/workflows/gitspiegel-trigger.yml # .github/workflows/release-99_notif-published.yml # bridges/snowbridge/README.md # bridges/snowbridge/runtime/test-common/src/lib.rs # bridges/snowbridge/scripts/contribute-upstream.sh # cumulus/parachains/runtimes/assets/test-utils/src/test_cases_over_bridge.rs # cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/snowbridge.rs # polkadot/node/subsystem-bench/src/approval/mod.rs # polkadot/node/subsystem-bench/src/availability/mod.rs # polkadot/node/subsystem-bench/src/cli.rs # polkadot/node/subsystem-bench/src/core/environment.rs # polkadot/node/subsystem-bench/src/subsystem-bench.rs
Snowfork · Feb 23, 2024 · 7bbe297 · 7bbe297
2 parents 4cede80 + 9bf1a5e
commit 7bbe297
Show file tree

Hide file tree

Showing 862 changed files with 16,744 additions and 9,847 deletions.
diff --git a/.config/taplo.toml b/.config/taplo.toml
@@ -2,10 +2,12 @@
 
 # ignore zombienet as they do some deliberate custom toml stuff
 exclude = [
+	"bridges/testing/**",
 	"cumulus/zombienet/**",
 	"polkadot/node/malus/integrationtests/**",
 	"polkadot/zombienet_tests/**",
 	"substrate/zombienet/**",
+	"target/**",
 ]
 
 # global rules

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -121,21 +121,17 @@ default:
   before_script:
     - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift  -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"'
     - chmod +x forklift
-    - mkdir .forklift
-    - cp $FL_FORKLIFT_CONFIG .forklift/config.toml
-    - export FORKLIFT_PACKAGE_SUFFIX=${CI_JOB_NAME/ [0-9 \/]*}
+    - mkdir ~/.forklift
+    - cp $FL_FORKLIFT_CONFIG ~/.forklift/config.toml
     - shopt -s expand_aliases
     - export PATH=$PATH:$(pwd)
     - |
       if [ "$FORKLIFT_BYPASS" != "true" ]; then
       echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"
       alias cargo="forklift cargo"
       fi
-    - ls -al
-    - rm -f forklift.sock
     #
     - echo "FL_FORKLIFT_VERSION ${FL_FORKLIFT_VERSION}"
-    - echo "FORKLIFT_PACKAGE_SUFFIX $FORKLIFT_PACKAGE_SUFFIX"
 
 .common-refs:
   rules:

diff --git a/.gitlab/check-each-crate.py b/.gitlab/check-each-crate.py
@@ -55,7 +55,7 @@
 
 	print(f"Checking {crates[crate][0]}", file=sys.stderr)
 
-	res = subprocess.run(["cargo", "check", "--locked"], cwd = crates[crate][1])
+	res = subprocess.run(["forklift", "cargo", "check", "--locked"], cwd = crates[crate][1])
 
 	if res.returncode != 0:
 		sys.exit(1)
diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml
@@ -314,8 +314,9 @@ build-linux-substrate:
     # tldr: we need to checkout the branch HEAD explicitly because of our dynamic versioning approach while building the substrate binary
     # see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589
     - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+    - !reference [.forklift-cache, before_script]
   script:
-    - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p staging-node-cli
+    - time WASM_BUILD_NO_COLOR=1 cargo build --locked --release -p staging-node-cli
     - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate
     - echo -n "Substrate version = "
     - if [ "${CI_COMMIT_TAG}" ]; then
@@ -334,6 +335,7 @@ build-runtimes-polkavm:
   extends:
     - .docker-env
     - .common-refs
+    - .run-immediately
   script:
     - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p minimal-runtime
     - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p westend-runtime
@@ -352,9 +354,10 @@ build-runtimes-polkavm:
     CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
   before_script:
     - mkdir -p ./artifacts/subkey
+    - !reference [.forklift-cache, before_script]
   script:
     - cd ./substrate/bin/utils/subkey
-    - SKIP_WASM_BUILD=1 time cargo build --locked --release
+    - time SKIP_WASM_BUILD=1 cargo build --locked --release
     # - cd -
     # - mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/.
     # - echo -n "Subkey version = "
@@ -393,3 +396,18 @@ build-subkey-linux:
 #   after_script: [""]
 #   tags:
 #     - osx
+
+# bridges
+
+# we need some non-binary artifacts in our bridges+zombienet image
+prepare-bridges-zombienet-artifacts:
+  stage: build
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+    - .collect-artifacts
+  before_script:
+    - mkdir -p ./artifacts/bridges-polkadot-sdk/bridges
+  script:
+    - cp -r bridges/testing ./artifacts/bridges-polkadot-sdk/bridges/testing
diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml
@@ -133,6 +133,7 @@ check-runtime-migration-westend:
     WASM: "westend_runtime.compact.compressed.wasm"
     URI: "wss://westend-try-runtime-node.parity-chains.parity.io:443"
     SUBCOMMAND_EXTRA_ARGS: "--no-weight-warnings"
+  allow_failure: true
 
 check-runtime-migration-rococo:
   stage: check

diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml
@@ -66,6 +66,8 @@ publish-rustdoc:
 # note: images are used not only in zombienet but also in rococo, wococo and versi
 .build-push-image:
   image: $BUILDAH_IMAGE
+  extends:
+    - .zombienet-refs
   variables:
     DOCKERFILE: "" # docker/path-to.Dockerfile
     IMAGE_NAME: "" # docker.io/paritypr/image_name
@@ -77,6 +79,7 @@ publish-rustdoc:
       --build-arg VCS_REF="${CI_COMMIT_SHA}"
       --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
       --build-arg IMAGE_NAME="${IMAGE_NAME}"
+      --build-arg ZOMBIENET_IMAGE="${ZOMBIENET_IMAGE}"
       --tag "$IMAGE_NAME:${DOCKER_IMAGES_VERSION}"
       --file ${DOCKERFILE} .
     - echo "$PARITYPR_PASS" |
@@ -163,3 +166,22 @@ build-push-image-substrate-pr:
   variables:
     DOCKERFILE: "docker/dockerfiles/substrate_injected.Dockerfile"
     IMAGE_NAME: "docker.io/paritypr/substrate"
+
+# unlike other images, bridges+zombienet image is based on Zombienet image that pulls required binaries
+# from other fresh images (polkadot and cumulus)
+build-push-image-bridges-zombienet-tests:
+  stage: publish
+  extends:
+    - .kubernetes-env
+    - .common-refs
+    - .build-push-image
+  needs:
+    - job: build-linux-stable
+      artifacts: true
+    - job: build-linux-stable-cumulus
+      artifacts: true
+    - job: prepare-bridges-zombienet-artifacts
+      artifacts: true
+  variables:
+    DOCKERFILE: "docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile"
+    IMAGE_NAME: "docker.io/paritypr/bridges-zombienet-tests"
diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml
@@ -224,6 +224,7 @@ cargo-check-benches:
       git merge --verbose --no-edit FETCH_HEAD;
       fi
       fi'
+    - !reference [.forklift-cache, before_script]
   parallel: 2
   script:
     - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA

diff --git a/.gitlab/pipeline/zombienet.yml b/.gitlab/pipeline/zombienet.yml
@@ -10,3 +10,5 @@ include:
   - .gitlab/pipeline/zombienet/cumulus.yml
   # polkadot tests
   - .gitlab/pipeline/zombienet/polkadot.yml
+  # bridges tests
+  - .gitlab/pipeline/zombienet/bridges.yml
diff --git a/.gitlab/pipeline/zombienet/bridges.yml b/.gitlab/pipeline/zombienet/bridges.yml
@@ -0,0 +1,63 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "zombienet" stage for bridges
+
+# common settings for all zombienet jobs
+.zombienet-bridges-common:
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  rules:
+    # Docker images have different tag in merge queues
+    - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/
+      variables:
+        DOCKER_IMAGES_VERSION: ${CI_COMMIT_SHORT_SHA}
+    - !reference [.build-refs, rules]
+  before_script:
+    - echo "Zombienet Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${GH_DIR}"
+    - echo "${LOCAL_DIR}"
+    - ls "${LOCAL_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${BRIDGES_ZOMBIENET_TESTS_IMAGE}":${BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG}
+    - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+  stage: zombienet
+  image: "${BRIDGES_ZOMBIENET_TESTS_IMAGE}:${BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG}"
+  needs:
+    - job: build-push-image-bridges-zombienet-tests
+      artifacts: true
+  variables:
+    BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG: ${DOCKER_IMAGES_VERSION}
+    BRIDGES_ZOMBIENET_TESTS_IMAGE: "docker.io/paritypr/bridges-zombienet-tests"
+    GH_DIR: "https://github.com/paritytech/polkadot-sdk/tree/${CI_COMMIT_SHA}/bridges/testing"
+    LOCAL_DIR: "/builds/parity/mirrors/polkadot-sdk/bridges/testing"
+    FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
+    RUN_IN_CONTAINER: "1"
+  artifacts:
+    name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}_zombienet_bridge_tests"
+    when: always
+    expire_in: 2 days
+    paths:
+      - ./zombienet-logs
+  after_script:
+    - mkdir -p ./zombienet-logs
+    # copy general logs
+    - cp -r /tmp/bridges-tests-run-*/logs/* ./zombienet-logs/
+    # copy logs of rococo nodes
+    - cp -r /tmp/bridges-tests-run-*/bridge_hub_rococo_local_network/*.log ./zombienet-logs/
+    # copy logs of westend nodes
+    - cp -r /tmp/bridges-tests-run-*/bridge_hub_westend_local_network/*.log ./zombienet-logs/
+
+zombienet-bridges-0001-asset-transfer-works:
+  extends:
+    - .zombienet-bridges-common
+  script:
+    - /home/nonroot/bridges-polkadot-sdk/bridges/testing/run-new-test.sh 0001-asset-transfer --docker
+    - echo "Done"
+
+zombienet-bridges-0002-mandatory-headers-synced-while-idle:
+  extends:
+    - .zombienet-bridges-common
+  script:
+    - /home/nonroot/bridges-polkadot-sdk/bridges/testing/run-new-test.sh 0002-mandatory-headers-synced-while-idle --docker
+    - echo "Done"