Merge pull request #1606 from OffchainLabs/contracts-develop-merge-ma…

…ster

Merge master into contracts-develop

.github/workflows/arbitrator-ci.yml

@@ -18,7 +18,7 @@ on:
 env:
   RUST_BACKTRACE: 1
   RUSTFLAGS: -Dwarnings
-  WABT_VERSION: 1.0.24
+  WABT_VERSION: 1.0.32
 
 jobs:
   coverage:
@@ -33,17 +33,28 @@ jobs:
       - name: Install Ubuntu dependencies
         run: |
           sudo add-apt-repository -y ppa:ethereum/ethereum
-          sudo add-apt-repository -y ppa:longsleep/golang-backports
           sudo apt-get update && sudo apt-get install -y \
-            build-essential cmake nodejs ethereum lld-14 golang-go libudev-dev
+            build-essential cmake ethereum lld-14 libudev-dev
           sudo ln -s /usr/bin/wasm-ld-14 /usr/local/bin/wasm-ld
 
+      - name: Install go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.x
+
+      - name: Setup nodejs
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+          cache: 'yarn'
+          cache-dependency-path: '**/yarn.lock'
+
       - name: Install rust stable
         uses: actions-rs/toolchain@v1
         id: install-rust
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "stable"
           override: true
           components: 'llvm-tools-preview, rustfmt, clippy'
 

.github/workflows/ci.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Install go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18.x
+          go-version: 1.19.x
 
       - name: Install wasm-ld
         run: |
@@ -55,22 +55,22 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "stable"
           target: wasm32-unknown-unknown
 
       - name: Install rust wasm32-wasi
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "stable"
           target: wasm32-wasi
 
       - name: Install rust stable
         uses: actions-rs/toolchain@v1
         id: install-rust
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "stable"
           override: true
 
       - name: Cache Build Products
@@ -112,21 +112,9 @@ jobs:
         if: steps.cache-cbrotli.outputs.cache-hit != 'true'
         run: ./build-brotli.sh -l
 
-      - name: Setup emsdk
+      - name: Build cbrotli-wasm in docker
         if: steps.cache-cbrotli.outputs.cache-hit != 'true'
-        uses: mymindstorm/setup-emsdk@v11
-        with:
-          # Make sure to set a version number!
-          version: 3.1.6
-          # This is the name of the cache folder.
-          # The cache folder will be placed in the build directory,
-          #  so make sure it doesn't conflict with anything!
-          actions-cache-folder: 'emsdk-cache'
-          no-cache: true
-
-      - name: Build cbrotli-wasm
-        if: steps.cache-cbrotli.outputs.cache-hit != 'true'
-        run: ./build-brotli.sh -w
+        run: ./build-brotli.sh -w -d
 
       - name: Build
         run: make build test-go-deps -j
@@ -145,20 +133,24 @@ jobs:
         run: |
           mkdir -p target/tmp
           echo "TMPDIR=$(pwd)/target/tmp" >> "$GITHUB_ENV"
-          echo "GOMEMLIMIT=6GB" >> "$GITHUB_ENV"
+          echo "GOMEMLIMIT=6GiB" >> "$GITHUB_ENV"
           echo "GOGC=80" >> "$GITHUB_ENV"
 
       - name: run tests without race detection
         if: matrix.test-mode == 'defaults'
-        run: gotestsum --format short-verbose -- ./... -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -parallel=8
+        run: |
+          packages=`go list ./...`
+          gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -parallel=8
 
       - name: run tests with race detection
         if: matrix.test-mode == 'race'
-        run: gotestsum --format short-verbose -- ./... -race -parallel=8
+        run:  |
+          packages=`go list ./...`
+          gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -race -parallel=8
 
       - name: run redis tests
         if: matrix.test-mode == 'defaults'
-        run: gotestsum --format short-verbose -- -p 1 -run TestRedis -tags redistest ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./...
+        run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./...
 
       - name: run challenge tests
         if: matrix.test-mode == 'challenge'

.github/workflows/codeql-analysis.yml

@@ -30,6 +30,7 @@ jobs:
       security-events: write
     env:
       CODEQL_EXTRACTOR_GO_BUILD_TRACING: 'on'
+      WABT_VERSION: 1.0.32
 
     strategy:
       fail-fast: false
@@ -52,7 +53,7 @@ jobs:
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
-        
+
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         # queries: security-extended,security-and-quality
         config-file: ./.github/codeql/codeql-config.yml
@@ -67,14 +68,14 @@ jobs:
     - name: Install go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.18.x
+        go-version: 1.19.x
 
     - name: Install rust stable
       uses: actions-rs/toolchain@v1
       id: install-rust
       with:
         profile: minimal
-        toolchain: stable
+        toolchain: "stable"
         override: true
 
     - name: Cache Rust Build Products
@@ -90,6 +91,13 @@ jobs:
         key: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-min-${{ hashFiles('arbitrator/Cargo.lock') }}
         restore-keys: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-
 
+    - name: Cache wabt build
+      id: cache-wabt
+      uses: actions/cache@v3
+      with:
+        path: ~/wabt-prefix
+        key: ${{ runner.os }}-wabt-codeql-${{ env.WABT_VERSION }}
+
     - name: Cache cbrotli
       uses: actions/cache@v3
       id: cache-cbrotli
@@ -122,7 +130,7 @@ jobs:
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
     #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
     # - run: |

Dockerfile

@@ -42,12 +42,15 @@ FROM wasm-base as wasm-libs-builder
 	# clang / lld used by soft-float wasm
 RUN apt-get install -y clang=1:11.0-51+nmu5 lld=1:11.0-51+nmu5
     # pinned rust 1.65.0
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.65.0 --target x86_64-unknown-linux-gnu wasm32-unknown-unknown wasm32-wasi
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.68.2 --target x86_64-unknown-linux-gnu wasm32-unknown-unknown wasm32-wasi
 COPY ./Makefile ./
 COPY arbitrator/wasm-libraries arbitrator/wasm-libraries
 COPY --from=brotli-wasm-export / target/
 RUN . ~/.cargo/env && NITRO_BUILD_IGNORE_TIMESTAMPS=1 RUSTFLAGS='-C symbol-mangling-version=v0' make build-wasm-libs
 
+FROM scratch as wasm-libs-export
+COPY --from=wasm-libs-builder /workspace/ /
+
 FROM wasm-base as wasm-bin-builder
     # pinned go version
 RUN curl -L https://golang.org/dl/go1.19.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf -
@@ -56,6 +59,7 @@ COPY ./arbcompress ./arbcompress
 COPY ./arbos ./arbos
 COPY ./arbstate ./arbstate
 COPY ./arbutil ./arbutil
+COPY ./gethhook ./gethhook
 COPY ./blsSignatures ./blsSignatures
 COPY ./cmd/replay ./cmd/replay
 COPY ./das/dastree ./das/dastree
@@ -74,7 +78,7 @@ COPY --from=contracts-builder workspace/contracts/build/contracts/src/precompile
 COPY --from=contracts-builder workspace/.make/ .make/
 RUN PATH="$PATH:/usr/local/go/bin" NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-wasm-bin
 
-FROM rust:1.65-slim-bullseye as prover-header-builder
+FROM rust:1.68-slim-bullseye as prover-header-builder
 WORKDIR /workspace
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
@@ -90,11 +94,11 @@ RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-prover-header
 FROM scratch as prover-header-export
 COPY --from=prover-header-builder /workspace/target/ /
 
-FROM rust:1.65-slim-bullseye as prover-builder
+FROM rust:1.68-slim-bullseye as prover-builder
 WORKDIR /workspace
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install -y make wget gpg software-properties-common zlib1g-dev libstdc++-10-dev
+    apt-get install -y make wget gpg software-properties-common zlib1g-dev libstdc++-10-dev wabt
 RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
     add-apt-repository 'deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-12 main' && \
     apt-get update && \

LICENSE

@@ -10,19 +10,27 @@ Parameters
 Licensor:             Offchain Labs
 
 Licensed Work:        Arbitrum Nitro
-                      The Licensed Work is (c) 2021-2022 Offchain Labs
+                      The Licensed Work is (c) 2021-2023 Offchain Labs
 
 Additional Use Grant: You may use the Licensed Work in a production environment solely
                       to provide a point of interface to permit end users or applications
                       utilizing the Covered Arbitrum Chains to interact and query the
                       state of a Covered Arbitrum Chain, including without limitation
                       validating the correctness of the posted chain state. For purposes
                       of this Additional Use Grant, the "Covered Arbitrum Chains" are
-                      Arbitrum One (chainid:42161), Arbitrum Nova (chainid:42170),
+                      means (a) Arbitrum One (chainid:42161), Arbitrum Nova (chainid:42170),
                       Arbitrum Rinkeby testnet/Rinkarby (chainid:421611), and
-                      Arbitrum Nitro Goerli testnet (chainid:421613).
+                      Arbitrum Nitro Goerli testnet (chainid:421613) (b) any future
+                      blockchains authorized to be designated as Covered Arbitrum Chains
+                      by the decentralized autonomous organization governing the Arbitrum
+                      network; and (c) any “Layer 3” Arbitrum-based blockchain that is built
+                      on and settles to another Covered Arbitrum Chain.
 
-Change Date:          
+
+
+
+
+Change Date:          Dec 31, 2027
 
 Change License:       Apache License Version 2.0
 
@@ -103,4 +111,4 @@ Notice
 
 The Business Source License (this document, or the "License") is not an Open
 Source license. However, the Licensed Work will eventually be made available
-under an Open Source License, as stated in this License.
+under an Open Source License, as stated in this License.

Makefile

@@ -66,6 +66,11 @@ WASI_SYSROOT?=/opt/wasi-sdk/wasi-sysroot
 arbitrator_wasm_lib_flags_nogo=$(patsubst %, -l %, $(arbitrator_wasm_libs_nogo))
 arbitrator_wasm_lib_flags=$(patsubst %, -l %, $(arbitrator_wasm_libs))
 
+rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/*.toml)
+
+prover_src = arbitrator/prover/src
+rust_prover_files = $(wildcard $(prover_src)/*.* $(prover_src)/*/*.* arbitrator/prover/*.toml) $(rust_arbutil_files)
+
 jit_dir = arbitrator/jit
 jit_files = $(wildcard $(jit_dir)/*.toml $(jit_dir)/*.rs $(jit_dir)/src/*.rs) $(rust_arbutil_files)
 
@@ -124,7 +129,7 @@ test-go-challenge: test-go-deps
 	@printf $(done)
 
 test-go-redis: test-go-deps
-	go test -p 1 -run TestRedis -tags redistest ./system_tests/... ./arbnode/...
+	TEST_REDIS=redis://localhost:6379/0 go test -p 1 -run TestRedis ./system_tests/... ./arbnode/...
 	@printf $(done)
 
 test-gen-proofs: \
@@ -182,12 +187,12 @@ $(replay_wasm): $(DEP_PREDICATE) $(go_source) .make/solgen
 	GOOS=js GOARCH=wasm go build -o $(output_root)/tmp/replay.wasm ./cmd/replay/...
 	if ! diff -qN $(output_root)/tmp/replay.wasm $@ > /dev/null; then cp $(output_root)/tmp/replay.wasm $@; fi
 
-$(arbitrator_prover_bin): $(DEP_PREDICATE) arbitrator/prover/src/*.rs arbitrator/prover/Cargo.toml
+$(arbitrator_prover_bin): $(DEP_PREDICATE) $(rust_prover_files)
 	mkdir -p `dirname $(arbitrator_prover_bin)`
 	cargo build --manifest-path arbitrator/Cargo.toml --release --bin prover ${CARGOFLAGS}
 	install arbitrator/target/release/prover $@
 
-$(arbitrator_prover_lib): $(DEP_PREDICATE) arbitrator/prover/src/*.rs arbitrator/prover/Cargo.toml
+$(arbitrator_prover_lib): $(DEP_PREDICATE) $(rust_prover_files)
 	mkdir -p `dirname $(arbitrator_prover_lib)`
 	cargo build --manifest-path arbitrator/Cargo.toml --release --lib -p prover ${CARGOFLAGS}
 	install arbitrator/target/release/libprover.a $@
@@ -200,7 +205,7 @@ $(arbitrator_jit): $(DEP_PREDICATE) .make/cbrotli-lib $(jit_files)
 $(arbitrator_cases)/rust/target/wasm32-wasi/release/%.wasm: $(arbitrator_cases)/rust/src/bin/%.rs $(arbitrator_cases)/rust/src/lib.rs
 	cargo build --manifest-path $(arbitrator_cases)/rust/Cargo.toml --release --target wasm32-wasi --bin $(patsubst $(arbitrator_cases)/rust/target/wasm32-wasi/release/%.wasm,%, $@)
 
-$(arbitrator_cases)/go/main: $(arbitrator_cases)/go/main.go $(arbitrator_cases)/go/go.mod $(arbitrator_cases)/go/go.sum
+$(arbitrator_cases)/go/main: $(arbitrator_cases)/go/main.go
 	cd $(arbitrator_cases)/go && GOOS=js GOARCH=wasm go build main.go
 
 $(arbitrator_generated_header): $(DEP_PREDICATE) arbitrator/prover/src/lib.rs arbitrator/prover/src/utils.rs
@@ -231,7 +236,8 @@ arbitrator/wasm-libraries/soft-float/bindings64.o: $(DEP_PREDICATE) arbitrator/w
 $(output_root)/machines/latest/soft-float.wasm: $(DEP_PREDICATE) \
 		arbitrator/wasm-libraries/soft-float/bindings32.o \
 		arbitrator/wasm-libraries/soft-float/bindings64.o \
-		arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a
+		arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a \
+		.make/wasm-lib
 	mkdir -p $(output_root)/machines/latest
 	wasm-ld \
 		arbitrator/wasm-libraries/soft-float/bindings32.o \
@@ -323,19 +329,23 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(arbitrator_pro
 	@touch $@
 
 .make/cbrotli-lib: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make
-	@printf "%btesting cbrotli local build exists. If this step fails, run ./build-brotli.sh -l%b\n" $(color_pink) $(color_reset)
-	test -f target/include/brotli/encode.h
-	test -f target/include/brotli/decode.h
-	test -f target/lib/libbrotlicommon-static.a
-	test -f target/lib/libbrotlienc-static.a
-	test -f target/lib/libbrotlidec-static.a
+	test -f target/include/brotli/encode.h || ./build-brotli.sh -l
+	test -f target/include/brotli/decode.h || ./build-brotli.sh -l
+	test -f target/lib/libbrotlicommon-static.a || ./build-brotli.sh -l
+	test -f target/lib/libbrotlienc-static.a || ./build-brotli.sh -l
+	test -f target/lib/libbrotlidec-static.a || ./build-brotli.sh -l
 	@touch $@
 
 .make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make
-	@printf "%btesting cbrotli wasm build exists. If this step fails, run ./build-brotli.sh -w%b\n" $(color_pink) $(color_reset)
-	test -f target/lib-wasm/libbrotlicommon-static.a
-	test -f target/lib-wasm/libbrotlienc-static.a
-	test -f target/lib-wasm/libbrotlidec-static.a
+	test -f target/lib-wasm/libbrotlicommon-static.a || ./build-brotli.sh -w -d
+	test -f target/lib-wasm/libbrotlienc-static.a || ./build-brotli.sh -w -d
+	test -f target/lib-wasm/libbrotlidec-static.a || ./build-brotli.sh -w -d
+	@touch $@
+
+.make/wasm-lib: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make
+	test -f arbitrator/wasm-libraries/soft-float/bindings32.o || ./build-brotli.sh -f -d -t .
+	test -f arbitrator/wasm-libraries/soft-float/bindings64.o || ./build-brotli.sh -f -d -t .
+	test -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a || ./build-brotli.sh -f -d -t .
 	@touch $@
 
 .make:

README.md

@@ -22,6 +22,8 @@ advanced calldata compression, and more.
 
 See the live docs-site [here](https://developer.arbitrum.io/) (or [here](https://github.com/OffchainLabs/arbitrum-docs) for markdown docs source.)
 
+See [here](./audits) for security audit reports.
+
 The Nitro stack is built on several innovations. At its core is a new prover, which can do Arbitrum’s classic 
 interactive fraud proofs over WASM code. That means the L2 Arbitrum engine can be written and compiled using 
 standard languages and tools, replacing the custom-designed language and compiler used in previous Arbitrum