Build system

.github/workflows/relayer-ci.yml

@@ -21,39 +21,55 @@ jobs:
         ]
         include:
           - ci_step: "lint"
-            command: "make lint-go"
+            command: "coreumbridge-xrpl-builder lint"
+            go-cache: true
+            wasm-cache: false
             linter-cache: true
             docker-cache: false
-            build-contract: false
           - ci_step: "test"
-            command: "make test-relayer"
+            command: "coreumbridge-xrpl-builder test"
+            go-cache: true
+            wasm-cache: false
             linter-cache: false
             docker-cache: false
-            build-contract: false
+          # FIXME (wojciech): fuzz tests still use `make` because we need to support them in crust first
           - ci_step: "fuzz-test"
             command: "make test-fuzz"
+            go-cache: true
+            wasm-cache: false
             linter-cache: false
             docker-cache: false
-            build-contract: false
           - ci_step: "integration tests contract"
-            command: "make restart-dev-env && make test-single-integration TESTS_DIR=integration-tests/contract"
+            command: |
+              coreumbridge-xrpl-builder build/contract build/integration-tests/contract images
+              coreum-builder images
+              crust znet test --test-groups=coreumbridge-xrpl-contract --timeout-commit 0.5s
+            go-cache: true
+            wasm-cache: true
             linter-cache: false
             docker-cache: true
-            build-contract: true
           - ci_step: "integration tests processes"
-            command: "make restart-dev-env && make test-single-integration TESTS_DIR=integration-tests/processes"
+            command: |
+              coreumbridge-xrpl-builder build/contract build/integration-tests/processes images
+              coreum-builder images
+              crust znet test --test-groups=coreumbridge-xrpl-processes --timeout-commit 0.5s
+            go-cache: true
+            wasm-cache: true
             linter-cache: false
             docker-cache: true
-            build-contract: true
           - ci_step: "integration tests xrpl"
-            command: "make restart-dev-env && make test-single-integration TESTS_DIR=integration-tests/xrpl"
+            command: |
+              coreumbridge-xrpl-builder build/contract build/integration-tests/xrpl images
+              coreum-builder images
+              crust znet test --test-groups=coreumbridge-xrpl-xrpl --timeout-commit 0.5s
+            go-cache: true
+            wasm-cache: true
             linter-cache: false
             docker-cache: true
-            build-contract: true
 
     runs-on: ubuntu-22.04
     steps:
-      - name: Go version used to build crust tool
+      - name: Go version used to build builder
         run: go version
       - name: Checkout coreumbridge-xrpl
         uses: actions/checkout@v4
@@ -65,65 +81,71 @@ jobs:
         with:
           repository: CoreumFoundation/crust
           path: crust
-          ref: bfeaa9ae4eace8b2d66e718ad50a03acd59d182c
+          # FIXME (wojciech): change to master once https://reviewable.io/reviews/CoreumFoundation/crust/365
+          # is merged
+          ref: e8b4330cd4fd6fe9841b6324d66910dbcec45451
       - name: Checkout coreum
         uses: actions/checkout@v4
         with:
           repository: CoreumFoundation/coreum
           path: coreum
           ref: 4695973601fb222a90677c645e975f6a264238b6
-      - name: Get crust-coreum cache key
-        id: crust-coreum-cache-key
+      - name: Set up build system
         run: |
-          cd ${{ github.workspace }}/crust
-          CRUST_HASH=$(git rev-parse HEAD)
-          cd ${{ github.workspace }}/coreum
-          COREM_HASH=$(git rev-parse HEAD)
-          echo "key=$CRUST_HASH-$COREM_HASH" >> $GITHUB_OUTPUT
-        shell: bash
-      - name: Set up crust binaries
-        run: echo "$(pwd)/crust/bin" >> $GITHUB_PATH
-      - name: Set up crust cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/crust/
-            ${{ github.workspace }}/crust/bin/.cache
-          key: ${{ runner.os }}-crust-cache-${{ steps.crust-coreum-cache-key.outputs.key }}
-      - name: Set up coreum binaries
-        run: echo "$(pwd)/coreum/bin" >> $GITHUB_PATH
-      - name: Set up go cache
-        uses: actions/cache@v4
+          echo "$(pwd)/coreumbridge-xrpl/bin" >> $GITHUB_PATH
+          echo "$(pwd)/coreum/bin" >> $GITHUB_PATH
+          echo "$(pwd)/crust/bin" >> $GITHUB_PATH
+          coreumbridge-xrpl/bin/coreumbridge-xrpl-builder build/me
+      - name: Retrieve go version
+        id: goversion
+        run: echo "GO_VERSION=$(coreumbridge-xrpl/bin/go version)" >> $GITHUB_OUTPUT
+      - name: Print go version
+        run: echo ${{ steps.goversion.outputs.GO_VERSION }}
+      - name: Setup go cache
+        uses: actions/cache@v3
+        if: ${{ matrix.go-cache }}
         with:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go-cache-${{ hashFiles('**/go.sum') }}
-      - name: Set up linter cache
-        uses: actions/cache@v4
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ steps.goversion.outputs.GO_VERSION }}
+      - name: Setup WASM cache
+        uses: actions/cache@v3
+        if: ${{ matrix.wasm-cache }}
+        with:
+          # we need to exclude code-hashes.json from the rest, since the invalidation condition for it is not
+          # same as WASM cache, but it is same as smart contract artifacts.
+          path: |
+            ~/.cache/crust/wasm
+            !/.cache/crust/wasm/code-hashes.json
+          key: ${{ runner.os }}-wasm-cache-${{ hashFiles('~/.cache/crust/wasm/**/*.rs') }}
+      - name: Setup linter cache
+        uses: actions/cache@v3
         if: ${{ matrix.linter-cache }}
         with:
           path: ~/.cache/golangci-lint
-          key: ${{ runner.os }}-linter-cache
-      - name: Set up wasm cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ github.workspace }}/coreumbridge-xrpl/contract/target/
-          key: ${{ runner.os }}-wasm-cache-${{ hashFiles('**/Cargo.lock') }}
-        if: ${{ matrix.build-contract }}
-      - name: Set up docker cache
+          key: ${{ runner.os }}-linter-cache-2-${{ steps.goversion.outputs.GO_VERSION }}
+      - name: Get Date
+        id: get-year-week
+        run: |
+          echo "date=$(/bin/date -u "+%Y-%U")" >> $GITHUB_OUTPUT
+        shell: bash
+      - name: Set docker cache
         uses: satackey/action-docker-layer-caching@v0.0.11
         if: ${{ matrix.docker-cache }}
         continue-on-error: true
         with:
-          key: ${{ runner.os }}-docker-cache-${{ steps.crust-coreum-cache-key.outputs.key }}
-      - name: Build contract
-        run: make build-contract
-        working-directory: ./coreumbridge-xrpl
-        if: ${{ matrix.build-contract }}
-      - name: Build dev environment
-        run: make build-dev-env
-        working-directory: ./coreumbridge-xrpl
+          key: ${{ runner.os }}-docker-v3-${{ steps.get-year-week.outputs.date }} # year-week key
+      - name: Setup smart contract build cache
+        uses: actions/cache@v3
+        with:
+          # we need to cache code-hashes.json under the same key as the artifacts, since the invalidation condition
+          # for both of them are the same.
+          path: |
+            ${{ github.workspace }}/coreumbridge-xrpl/contract/artifacts/* 
+            ~/.cache/crust/wasm/code-hashes.json
+          key: ${{ runner.os }}-cache-smart-contracts-${{ hashFiles('./coreumbridge-xrpl/contract/**/*.rs') }}
+        if: ${{ matrix.wasm-cache }}
       - name: Run ${{ matrix.ci_step }}
         working-directory: ./coreumbridge-xrpl
         run: ${{ matrix.command }}

.gitignore

@@ -1,5 +1,6 @@
 .idea
 .vscode
-build
 vendor
 *.iml
+/coverage
+/bin

Makefile

@@ -53,7 +53,6 @@ build-contract:
       -v $(CONTRACT_DIR)/target:/usr/local/cargo/registry \
       cosmwasm/optimizer:0.15.0
 	mkdir -p $(BUILD_DIR)
-	cp $(CONTRACT_DIR)/artifacts/coreumbridge_xrpl.wasm $(BUILD_DIR)/coreumbridge_xrpl.wasm
 
 .PHONY: build-dev-contract
 build-dev-contract:

bin/coreumbridge-xrpl-builder

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# go to root dir of repository
+pushd "$(dirname "${BASH_SOURCE[0]}")/.." > /dev/null
+
+VERSION=$(git rev-parse --short HEAD)
+REPO=$(pwd)
+BUILD_BIN="$REPO/bin/.cache/build-$VERSION"
+
+if [ ! -f "$BUILD_BIN" ]; then
+  rm -f ./bin/.cache/build-*
+
+  pushd build > /dev/null
+  go build -trimpath -o "$BUILD_BIN" ./cmd
+  popd > /dev/null
+
+  "$BUILD_BIN" build/me
+fi
+
+popd > /dev/null
+
+exec "$BUILD_BIN" "$@"

build/bridge/contract.go

@@ -0,0 +1,150 @@
+package bridge
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	"golang.org/x/mod/sumdb/dirhash"
+
+	"github.com/CoreumFoundation/coreum-tools/pkg/build"
+	"github.com/CoreumFoundation/coreum-tools/pkg/logger"
+	"github.com/CoreumFoundation/crust/build/rust"
+	"github.com/CoreumFoundation/crust/build/tools"
+)
+
+// BuildSmartContract builds bridge smart contract.
+func BuildSmartContract(ctx context.Context, deps build.DepsFunc) error {
+	deps(CompileSmartContract(filepath.Join(repoPath, "contract")))
+	return nil
+}
+
+// FIXME (wojciech): Code copied from coreum, let's move it to crust
+
+// CompileSmartContract returns function compiling smart contract.
+func CompileSmartContract(codeDirPath string) build.CommandFunc {
+	return func(ctx context.Context, deps build.DepsFunc) error {
+		log := logger.Get(ctx)
+		log.Info("Compiling WASM smart contract", zap.String("path", codeDirPath))
+
+		codeDirAbsPath, err := filepath.Abs(codeDirPath)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+
+		contractSrcHash, err := computeContractSrcHash(codeDirAbsPath)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+
+		wasmCachePath := filepath.Join(tools.CacheDir(), "wasm")
+		if err := os.MkdirAll(wasmCachePath, 0o700); err != nil {
+			return errors.WithStack(err)
+		}
+
+		codeHashesFile, err := os.OpenFile(filepath.Join(wasmCachePath, "code-hashes.json"), os.O_CREATE|os.O_RDWR, 0o700)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		defer codeHashesFile.Close()
+
+		codeHashesBytes, err := io.ReadAll(codeHashesFile)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		absPathHash := fmt.Sprintf("%x", sha256.Sum256([]byte(codeDirAbsPath)))
+
+		storedCodeHashes := make(map[string]string)
+		if len(codeHashesBytes) != 0 {
+			err := json.Unmarshal(codeHashesBytes, &storedCodeHashes)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+		}
+
+		if storedHash, ok := storedCodeHashes[absPathHash]; ok {
+			contractArtifactsHash, err := computeContractArtifactsHash(codeDirAbsPath)
+			if err != nil {
+				return err
+			}
+			codeHash := contractSrcHash + contractArtifactsHash
+			log.Info("Computed contract code hash", zap.String("hash", codeHash))
+			if codeHash == storedHash {
+				log.Info("No changes in the contract, skipping compilation.")
+				return nil
+			}
+		}
+
+		targetCachePath := filepath.Join(wasmCachePath, "targets", absPathHash)
+		if err := os.MkdirAll(targetCachePath, 0o700); err != nil {
+			return errors.WithStack(err)
+		}
+
+		registryCachePath := filepath.Join(wasmCachePath, "registry")
+		if err := os.MkdirAll(registryCachePath, 0o700); err != nil {
+			return errors.WithStack(err)
+		}
+
+		if err := rust.BuildSmartContract(ctx, deps, codeDirAbsPath); err != nil {
+			return err
+		}
+
+		contractArtifactsHash, err := computeContractArtifactsHash(codeDirAbsPath)
+		if err != nil {
+			return err
+		}
+		if contractArtifactsHash == "" {
+			return errors.New("artifacts folder doesn't exist after the contract building")
+		}
+
+		newCodeHash := contractSrcHash + contractArtifactsHash
+		storedCodeHashes[absPathHash] = newCodeHash
+		codeHashesBytes, err = json.Marshal(storedCodeHashes)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+
+		return replaceFileContent(codeHashesFile, codeHashesBytes)
+	}
+}
+
+func computeContractSrcHash(path string) (string, error) {
+	hash, err := dirhash.HashDir(filepath.Join(path, "src"), "", dirhash.Hash1)
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	return hash, nil
+}
+
+func computeContractArtifactsHash(path string) (string, error) {
+	hash, err := dirhash.HashDir(filepath.Join(path, "artifacts"), "", dirhash.Hash1)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", nil
+		}
+		return "", errors.WithStack(err)
+	}
+
+	return hash, nil
+}
+
+func replaceFileContent(codeHashesFile *os.File, codeHashesBytes []byte) error {
+	if err := codeHashesFile.Truncate(0); err != nil {
+		return errors.WithStack(err)
+	}
+	if _, err := codeHashesFile.Seek(0, 0); err != nil {
+		return errors.WithStack(err)
+	}
+	if _, err := codeHashesFile.Write(codeHashesBytes); err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}

build/bridge/generate.go

@@ -0,0 +1,13 @@
+package bridge
+
+import (
+	"context"
+
+	"github.com/CoreumFoundation/coreum-tools/pkg/build"
+	"github.com/CoreumFoundation/crust/build/golang"
+)
+
+// Generate regenerates everything in bridge.
+func Generate(ctx context.Context, deps build.DepsFunc) error {
+	return golang.Generate(ctx, repoPath, deps)
+}

build/bridge/image/Dockerfile.tmpl

@@ -0,0 +1,6 @@
+FROM --platform=$TARGETPLATFORM {{ .From }}
+
+ARG TARGETARCH
+COPY docker.linux.$TARGETARCH/{{ .Binary }} /{{ .Binary }}
+
+ENTRYPOINT ["/{{ .Binary }}"]