Merge branch 'cosmos:main' into fix/client-debug-helpmsg

cosmos · Sep 12, 2024 · 9ec34ff · 9ec34ff
2 parents 7b17292 + 4fe934e
commit 9ec34ff
Show file tree

Hide file tree

Showing 230 changed files with 10,007 additions and 13,369 deletions.
diff --git a/.github/scripts/get-rocksdb-version.sh b/.github/scripts/get-rocksdb-version.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# Search for rocksdb_version in makefile
+rocksdb_version=$(grep "rocksdb_version" ./scripts/build/build.mk | cut -d'=' -f2)
+
+if [[ -z "${rocksdb_version}" ]]; then
+    echo "Error: rocksdb_version not found in ./scripts/build/build.mk" >&2
+    exit 1
+else
+    echo "ROCKSDB_VERSION=${rocksdb_version}" >> "${GITHUB_ENV}"
+fi
diff --git a/.github/scripts/install-rocksdb-deps.sh b/.github/scripts/install-rocksdb-deps.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+sudo apt update && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev build-essential
diff --git a/.github/scripts/install-rocksdb.sh b/.github/scripts/install-rocksdb.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [ -z "$ROCKSDB_VERSION" ]; then
+  echo "ROCKSDB_VERSION is not set."
+  exit 1
+fi
+
+# Clone RocksDB repository
+git clone https://github.com/facebook/rocksdb.git /home/runner/rocksdb
+cd /home/runner/rocksdb || exit 1
+git checkout "${ROCKSDB_VERSION}"
+
+# Build shared library
+sudo make -j "$(nproc --all)" shared_lib
+sudo cp --preserve=links ./librocksdb.* /usr/local/lib/
+sudo cp -r ./include/rocksdb/ /usr/local/include/
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -21,12 +21,29 @@ jobs:
         go-arch: ["amd64", "arm", "arm64"]
     steps:
       - uses: actions/checkout@v4
-      - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - uses: actions/setup-go@v5
         with:
           go-version: "1.23"
           check-latest: true
+      - name: Get rocksdb version
+        run: ./.github/scripts/get-rocksdb-version.sh
+      - name: Fix permissions for cache
+        run: sudo chown $(whoami) /usr/local/lib /usr/local/include
+      - name: Restore rocksdb libraries cache
+        id: cache-rocksdb
+        if: matrix.go-arch == 'amd64'
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /usr/local/lib/librocksdb.*
+            /usr/local/include/rocksdb
+          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-${{ matrix.go-arch }}
+      - name: Install rocksdb deps
+        if: matrix.go-arch == 'amd64'
+        run: ./.github/scripts/install-rocksdb-deps.sh
+      - name: Install rocksdb
+        if: matrix.go-arch == 'amd64' && steps.cache-rocksdb.outputs.cache-hit != 'true'
+        run: ./.github/scripts/install-rocksdb.sh
         ###################
         #### Build App ####
         ###################
@@ -35,10 +52,8 @@ jobs:
       - name: Build Legacy
         run: GOARCH=${{ matrix.go-arch }} COSMOS_BUILD_OPTIONS=legacy make build
       - name: Build with rocksdb backend
-        if: |
-          env.GIT_DIFF &&
-          matrix.go-arch == 'amd64'
-        run: nix run . -- version --long
+        if: matrix.go-arch == 'amd64'
+        run: GOARCH=${{ matrix.go-arch }} COSMOS_BUILD_OPTIONS="rocksdb" make build
         ###################
         ## Build Tooling ##
         ###################

diff --git a/.github/workflows/cache-rocksdb.yml b/.github/workflows/cache-rocksdb.yml
@@ -0,0 +1,62 @@
+name: Cache rocksdb libraries
+on:
+  push:
+    paths:
+      - build.mk
+  schedule:
+    - cron: "*/15 * * * *" # Every 15 minutes
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+
+  check-cache-rocksdb:
+    name: Check existing cache
+    runs-on: ubuntu-latest
+    outputs:
+      cache-hit: ${{ steps.cache-rocksdb.outputs.cache-hit }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get rocksdb version
+        run: ./.github/scripts/get-rocksdb-version.sh
+
+      - name: Fix permissions for cache
+        run: sudo chown $(whoami) /usr/local/lib /usr/local/include
+
+      - name: Restore rocksdb libraries cache
+        id: cache-rocksdb
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /usr/local/lib/librocksdb.*
+            /usr/local/include/rocksdb
+          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
+
+
+  save-cache-rocksdb:
+    name: Build rocksdb libraries and save cache
+    runs-on: ubuntu-latest
+    needs: check-cache-rocksdb
+    if: needs.check-cache-rocksdb.outputs.cache-hit != 'true'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get rocksdb version
+        run: ./.github/scripts/get-rocksdb-version.sh
+
+      - name: Install rocksdb deps
+        run: ./.github/scripts/install-rocksdb-deps.sh
+      - name: Install rocksdb
+        run: ./.github/scripts/install-rocksdb.sh
+
+      - name: Saves rocksdb libraries cache
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            /usr/local/lib/librocksdb.*
+            /usr/local/include/rocksdb
+          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -8,14 +8,13 @@ on:
   merge_group:
 permissions:
   contents: read
+
 jobs:
   golangci:
     name: golangci-lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - uses: actions/setup-go@v5
         with:
           go-version: "1.23"
@@ -28,13 +27,30 @@ jobs:
             Makefile
             **/Makefile
             .golangci.yml
+      - name: Get rocksdb version
+        run: ./.github/scripts/get-rocksdb-version.sh
+      - name: Fix permissions for cache
+        run: sudo chown $(whoami) /usr/local/lib /usr/local/include
+      - name: Restore rocksdb libraries cache
+        id: cache-rocksdb
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /usr/local/lib/librocksdb.*
+            /usr/local/include/rocksdb
+          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
+      - name: Install rocksdb deps
+        run: ./.github/scripts/install-rocksdb-deps.sh
+      - name: Install rocksdb
+        if: steps.cache-rocksdb.outputs.cache-hit != 'true'
+        run: ./.github/scripts/install-rocksdb.sh
       - name: run linting (long)
         if: env.GIT_DIFF
         id: lint_long
         run: |
-          nix develop -c make lint
+          make lint
         env:
-          NIX: 1
+          ROCKSDB: 1
       - uses: technote-space/get-diff-action@v6.1.2
         if: steps.lint_long.outcome == 'skipped'
         with:
@@ -57,8 +73,8 @@ jobs:
       - name: run linting (short)
         if: steps.lint_long.outcome == 'skipped' && env.GIT_DIFF
         run: |
-          nix develop -c make lint
+          make lint
         env:
           GIT_DIFF: ${{ env.GIT_DIFF }}
           LINT_DIFF: 1
-          NIX: 1
+          ROCKSDB: 1
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -13,6 +13,7 @@ concurrency:
   group: ci-${{ github.ref }}-tests
   cancel-in-progress: true
 
+
 jobs:
   split-test-files:
     runs-on: ubuntu-latest
@@ -774,11 +775,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.20"
+          go-version: "1.23"
           check-latest: true
           cache: true
           cache-dependency-path: store/go.sum
@@ -789,11 +788,28 @@ jobs:
             store/**/*.go
             store/go.mod
             store/go.sum
+      - name: Get rocksdb version
+        run: ./.github/scripts/get-rocksdb-version.sh
+      - name: Fix permissions for cache
+        run: sudo chown $(whoami) /usr/local/lib /usr/local/include
+      - name: Restore rocksdb libraries cache
+        id: cache-rocksdb
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /usr/local/lib/librocksdb.*
+            /usr/local/include/rocksdb
+          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
+      - name: Install rocksdb deps
+        run: ./.github/scripts/install-rocksdb-deps.sh
+      - name: Install rocksdb
+        if: steps.cache-rocksdb.outputs.cache-hit != 'true'
+        run: ./.github/scripts/install-rocksdb.sh
       - name: tests
         if: env.GIT_DIFF
         run: |
           cd store
-          nix develop .. -c go test -mod=readonly -timeout 30m -coverprofile=coverage.out -covermode=atomic -tags='norace ledger test_ledger_mock rocksdb' ./...
+          go test -ldflags "-r /usr/local/lib" -mod=readonly -timeout 30m -coverprofile=coverage.out -covermode=atomic -tags='norace ledger test_ledger_mock rocksdb' ./...
       - name: sonarcloud
         if: ${{ env.GIT_DIFF && !github.event.pull_request.draft && env.SONAR_TOKEN != null }}
         uses: SonarSource/sonarcloud-github-action@master
@@ -809,8 +825,6 @@ jobs:
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
-      - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - uses: actions/setup-go@v5
         with:
           go-version: "1.23"
@@ -824,11 +838,28 @@ jobs:
             store/v2/**/*.go
             store/v2/go.mod
             store/v2/go.sum
+      - name: Get rocksdb version
+        run: ./.github/scripts/get-rocksdb-version.sh
+      - name: Fix permissions for cache
+        run: sudo chown $(whoami) /usr/local/lib /usr/local/include
+      - name: Restore rocksdb libraries cache
+        id: cache-rocksdb
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /usr/local/lib/librocksdb.*
+            /usr/local/include/rocksdb
+          key: ${{ runner.os }}-rocksdb-${{ env.ROCKSDB_VERSION }}-amd64
+      - name: Install rocksdb deps
+        run: ./.github/scripts/install-rocksdb-deps.sh
+      - name: Install rocksdb
+        if: steps.cache-rocksdb.outputs.cache-hit != 'true'
+        run: ./.github/scripts/install-rocksdb.sh
       - name: test & coverage report creation
         if: env.GIT_DIFF
         run: |
           cd store/v2
-          nix develop ../.. -c go test -mod=readonly -timeout 30m -coverprofile=coverage.out -covermode=atomic -tags='norace ledger test_ledger_mock rocksdb' ./...
+          go test -ldflags "-r /usr/local/lib" -mod=readonly -timeout 30m -coverprofile=coverage.out -covermode=atomic -tags='norace ledger test_ledger_mock rocksdb' ./...
       - name: sonarcloud
         if: ${{ env.GIT_DIFF && !github.event.pull_request.draft && env.SONAR_TOKEN != null }}
         uses: SonarSource/sonarcloud-github-action@master

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -49,6 +49,7 @@ Every module contains its own CHANGELOG.md. Please refer to the module you are i
 
 * (client) [#21436](https://github.com/cosmos/cosmos-sdk/pull/21436) Use `address.Codec` from client.Context in `tx.Sign`.
 * (internal) [#21412](https://github.com/cosmos/cosmos-sdk/pull/21412) Using unsafe.String and unsafe.SliceData.
+* (x/genutil) [#21249](https://github.com/cosmos/cosmos-sdk/pull/21249) Incremental JSON parsing for AppGenesis where possible.
 
 ### Bug Fixes
 

diff --git a/UPGRADING.md b/UPGRADING.md
@@ -277,6 +277,20 @@ If you are still using the legacy wiring, you must enable unordered transactions
 	}
 	```
 
+* Create or update the App's `Preblocker()` method to call the unordered tx
+  manager's `OnNewBlock()` method.
+
+	```go
+	...
+	app.SetPreblocker(app.PreBlocker)
+	...
+
+	func (app *SimApp) PreBlocker(ctx sdk.Context, req *abci.RequestFinalizeBlock) (*sdk.ResponsePreBlock, error) {
+		app.UnorderedTxManager.OnNewBlock(ctx.BlockTime())
+		return app.ModuleManager.PreBlock(ctx, req)
+	}
+	```
+
 * Create or update the App's `Close()` method to close the unordered tx manager.
   Note, this is critical as it ensures the manager's state is written to file
   such that when the node restarts, it can recover the state to provide replay
@@ -1040,7 +1054,7 @@ The `simapp` package **should not be imported in your own app**. Instead, you sh
 
 #### App Wiring
 
-SimApp's `app_v2.go` is using [App Wiring](https://docs.cosmos.network/main/build/building-apps/app-go-v2), the dependency injection framework of the Cosmos SDK.
+SimApp's `app_di.go` is using [App Wiring](https://docs.cosmos.network/main/build/building-apps/app-go-di), the dependency injection framework of the Cosmos SDK.
 This means that modules are injected directly into SimApp thanks to a [configuration file](https://github.com/cosmos/cosmos-sdk/blob/v0.47.0-rc1/simapp/app_config.go).
 The previous behavior, without the dependency injection framework, is still present in [`app.go`](https://github.com/cosmos/cosmos-sdk/blob/v0.47.0-rc1/simapp/app.go) and is not going anywhere.