Problem: pebbledb backend is not tested (#934)

* Problem: pebbledb backend is not tested

Solution:
- enable the backend and test in integration tests

* fix upgrade integration test

* fix versiondb integration test

* update dependency repo

* pin pebbledb to released version

CHANGELOG.md

@@ -26,6 +26,7 @@
 - [#890](https://github.com/crypto-org-chain/cronos/pull/890) optimize memiavl snapshot format.
 - [#904](https://github.com/crypto-org-chain/cronos/pull/904) Enable "dynamic-level-bytes" on new `application.db`.
 - [#924](https://github.com/crypto-org-chain/cronos/pull/924) memiavl support `Export` API.
+- [#934](https://github.com/crypto-org-chain/cronos/pull/934) Add pebbledb backend.
 
 *Feb 09, 2022*
 

go.mod

@@ -42,6 +42,8 @@ require (
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
 	github.com/99designs/keyring v1.2.1 // indirect
 	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect
+	github.com/DataDog/zstd v1.5.0 // indirect
+	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
 	github.com/VictoriaMetrics/metrics v1.23.1 // indirect
@@ -62,6 +64,11 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/cockroachdb/apd/v2 v2.0.2 // indirect
+	github.com/cockroachdb/errors v1.8.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230315005856-dcb60b9212f9 // indirect
+	github.com/cockroachdb/redact v1.0.8 // indirect
+	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
 	github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect
 	github.com/cometbft/cometbft-db v0.7.0 // indirect
 	github.com/confio/ics23/go v0.9.0 // indirect
@@ -130,7 +137,9 @@ require (
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
-	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263 // indirect
 	github.com/ledgerwatch/log/v3 v3.7.0 // indirect
 	github.com/lib/pq v1.10.6 // indirect
@@ -161,6 +170,7 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/regen-network/cosmos-proto v0.3.1 // indirect
 	github.com/rjeczalik/notify v0.9.1 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/rs/cors v1.8.2 // indirect
 	github.com/rs/zerolog v1.27.0 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
@@ -209,8 +219,9 @@ require (
 )
 
 replace (
-	// https://github.com/cometbft/cometbft-db/pull/42
-	github.com/cometbft/cometbft-db => github.com/crypto-org-chain/cometbft-db v0.0.0-20230306031617-b3e4fd3331c4
+	// the version used by cockroach v22.2.7 release
+	github.com/cockroachdb/pebble => github.com/cockroachdb/pebble v0.0.0-20230206180212-744ea7cc8f90
+	github.com/cometbft/cometbft-db => github.com/crypto-org-chain/cometbft-db v0.0.0-20230412133340-ac70df4b45f6
 	// Ref: https://forum.cosmos.network/t/ibc-security-advisory-dragonberry/7702
 	github.com/confio/ics23/go => github.com/confio/ics23/go v0.9.0
 	github.com/cosmos/cosmos-sdk => github.com/cosmos/cosmos-sdk v0.46.11
@@ -230,8 +241,7 @@ replace (
 	// use cometbft
 	github.com/tendermint/tendermint => github.com/cometbft/cometbft v0.34.27
 	// https://github.com/crypto-org-chain/tm-db/tree/release/v0.6.x
-	// still need this replacement for indirect dependencies on tm-db
-	github.com/tendermint/tm-db => github.com/crypto-org-chain/tm-db v0.6.8-0.20230118040049-14dc6b00a5b3
+	github.com/tendermint/tm-db => github.com/crypto-org-chain/tm-db v0.6.8-0.20230412133356-5e16d347f5b9
 
 	// TODO: remove after fixed https://github.com/cosmos/cosmos-sdk/issues/11364
 	github.com/zondax/hid => github.com/zondax/hid v0.9.0

gomod2nix.toml

@@ -34,6 +34,12 @@ schema = 3
   [mod."github.com/ChainSafe/go-schnorrkel"]
     version = "v0.0.0-20200405005733-88cbf1b4c40d"
     hash = "sha256-i8RXZemJGlSjBT35oPm0SawFiBoIU5Pkq5xp4n/rzCY="
+  [mod."github.com/DataDog/zstd"]
+    version = "v1.5.0"
+    hash = "sha256-i+vPadZcmSHCALA50FaADRmj6oqe1t5SIfqEbCxr9Ao="
+  [mod."github.com/HdrHistogram/hdrhistogram-go"]
+    version = "v1.1.2"
+    hash = "sha256-GZWNKwUmZLQ+krFvyFwBlGp/cLw/ihYuMuhKutuQIlA="
   [mod."github.com/StackExchange/wmi"]
     version = "v1.2.1"
     hash = "sha256-1BoEeWAWyebH+1mMuyPhWZut8nWHb6r73MgcqlGuUEY="
@@ -97,12 +103,28 @@ schema = 3
   [mod."github.com/cockroachdb/apd/v2"]
     version = "v2.0.2"
     hash = "sha256-UrPHkvqVF8V78+kXKmjTHl79XsgDBnqFsje5BMYh0E4="
+  [mod."github.com/cockroachdb/errors"]
+    version = "v1.8.1"
+    hash = "sha256-fpxeq5kDoP9NxP1XAWulmHfAKo0p0gDNj65YqPM318s="
+  [mod."github.com/cockroachdb/logtags"]
+    version = "v0.0.0-20190617123548-eb05cc24525f"
+    hash = "sha256-mdcKZsPfCiaTyxLRkZprVIijeZzbTD3J7WGb4zOQgn4="
+  [mod."github.com/cockroachdb/pebble"]
+    version = "v0.0.0-20230206180212-744ea7cc8f90"
+    hash = "sha256-P6TPgp2xCBM3IhqJ7sz9Jl7Gj41p63ZPrdR+8YzxMiE="
+    replaced = "github.com/cockroachdb/pebble"
+  [mod."github.com/cockroachdb/redact"]
+    version = "v1.0.8"
+    hash = "sha256-V/qBOGQUMbpPAQOpLTZk8MZiBr66Epml8snn9suCbs4="
+  [mod."github.com/cockroachdb/sentry-go"]
+    version = "v0.6.1-cockroachdb.2"
+    hash = "sha256-3C9tuGU6f2DOz6yPcOdUf1LRvCXFg+prfqAPob9Sz2E="
   [mod."github.com/coinbase/rosetta-sdk-go"]
     version = "v0.7.9"
     hash = "sha256-ZWIXIXcHGjeCNgMrpXymry8/8esDDauGFfF/+gEoO1Y="
   [mod."github.com/cometbft/cometbft-db"]
-    version = "v0.0.0-20230306031617-b3e4fd3331c4"
-    hash = "sha256-fGolPb6TBdk3+PPpVYpjSWAsfeDCG88X6H4edpTMgds="
+    version = "v0.0.0-20230412133340-ac70df4b45f6"
+    hash = "sha256-WOubYACTuk6OeHMSbcKIYbQWarWIljQUEvyX3hiXmq8="
     replaced = "github.com/crypto-org-chain/cometbft-db"
   [mod."github.com/confio/ics23/go"]
     version = "v0.9.0"
@@ -332,8 +354,14 @@ schema = 3
     version = "v1.0.0"
     hash = "sha256-xEd0mDBeq3eR/GYeXjoTVb2sPs8sTCosn5ayWkcgENI="
   [mod."github.com/klauspost/compress"]
-    version = "v1.15.11"
-    hash = "sha256-9MXm0TObg6DyqnYMIw3IChrorHc2ILf5djZYoM0e1J0="
+    version = "v1.15.15"
+    hash = "sha256-fMVJFz/P37vjZAf82nHjaItaB3gesOa+xV7uBuo3kmg="
+  [mod."github.com/kr/pretty"]
+    version = "v0.3.0"
+    hash = "sha256-5KfVas96NAU7pmN2ZOwQFFpUeagiJTqYKBLxq5aM5W4="
+  [mod."github.com/kr/text"]
+    version = "v0.2.0"
+    hash = "sha256-fadcWxZOORv44oak3jTxm6YcITcFxdGt4bpn869HxUE="
   [mod."github.com/ledgerwatch/erigon-lib"]
     version = "v0.0.0-20230210071639-db0e7ed11263"
     hash = "sha256-SKFGLsJV6G4xIQ5IU+qq9EY3v0/I8B/CTMcDOhXGfc4="
@@ -435,6 +463,9 @@ schema = 3
   [mod."github.com/rjeczalik/notify"]
     version = "v0.9.1"
     hash = "sha256-YLGNrHHM+mN4ElW/XWuylOnFrA/VjSY+eBuC4LN//5c="
+  [mod."github.com/rogpeppe/go-internal"]
+    version = "v1.8.1"
+    hash = "sha256-kXGiB1aTKIfgCKhZvSfIOyTL09EvMhzQEyhRmUhe7Ho="
   [mod."github.com/rs/cors"]
     version = "v1.8.2"
     hash = "sha256-FxBbQVb8zIoDGGaAJKfsYV0+POkh1tyX7P6A/Lk5Vsc="
@@ -488,8 +519,8 @@ schema = 3
     hash = "sha256-BbpfLcLRf6PwB1xmgkhTm9ckZekelrDUlXDX0/FSMU8="
     replaced = "github.com/cometbft/cometbft"
   [mod."github.com/tendermint/tm-db"]
-    version = "v0.6.8-0.20230118040049-14dc6b00a5b3"
-    hash = "sha256-kCe9nqzVgG+7ynuvYAGAOTCPESBnV28dDvRuAtsssbw="
+    version = "v0.6.8-0.20230412133356-5e16d347f5b9"
+    hash = "sha256-qWM+P4sAgvsl3m7fHAIQfav4DF4wZ6WfTP6mx3j1JrQ="
     replaced = "github.com/crypto-org-chain/tm-db"
   [mod."github.com/tidwall/btree"]
     version = "v1.5.0"

integration_tests/configs/cosmovisor.jsonnet

@@ -3,6 +3,7 @@ local config = import 'default.jsonnet';
 config {
   'cronos_777-1'+: {
     'app-config'+: {
+      'app-db-backend': 'rocksdb',
       'minimum-gas-prices': '100000000000basetcro',
       store:: super.store,
       streamers:: super.streamers,

integration_tests/configs/cosmovisor_gravity.jsonnet

@@ -3,6 +3,7 @@ local config = import 'default.jsonnet';
 config {
   'cronos_777-1'+: {
     'app-config'+: {
+      'app-db-backend': 'rocksdb',
       'minimum-gas-prices': '100000000000basetcro',
       store:: super.store,
       streamers:: super.streamers,

integration_tests/configs/default.jsonnet

@@ -9,7 +9,7 @@
       },
     },
     'app-config': {
-      'app-db-backend': 'rocksdb',
+      'app-db-backend': 'pebbledb',
       'minimum-gas-prices': '0basetcro',
       'index-events': ['ethereum_tx.ethereumTxHash'],
       'iavl-lazy-loading': true,

integration_tests/test_versiondb.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 
+import tomlkit
 from pystarport import ports
 
 from .network import Cronos
@@ -69,6 +70,10 @@ def test_versiondb_migration(cronos: Cronos):
         )
     )
 
+    # force app-db-backend to be rocksdb
+    patch_app_db_backend(cli0.data_dir / "config/app.toml", "rocksdb")
+    patch_app_db_backend(cli1.data_dir / "config/app.toml", "rocksdb")
+
     print("start all nodes")
     print(cronos.supervisorctl("start", "cronos_777-1-node0", "cronos_777-1-node1"))
     wait_for_port(ports.evmrpc_port(cronos.base_port(0)))
@@ -92,3 +97,9 @@ def test_versiondb_migration(cronos: Cronos):
             "value": 1000,
         },
     )
+
+
+def patch_app_db_backend(path, backend):
+    cfg = tomlkit.parse(path.read_text())
+    cfg["app-db-backend"] = backend
+    path.write_text(tomlkit.dumps(cfg))

memiavl/go.mod

@@ -21,11 +21,18 @@ require (
 
 require (
 	cosmossdk.io/errors v1.0.0-beta.7 // indirect
+	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/VictoriaMetrics/metrics v1.23.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
 	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.8.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230117234908-f91ad392a02b // indirect
+	github.com/cockroachdb/redact v1.0.8 // indirect
+	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
 	github.com/confio/ics23/go v0.9.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
@@ -44,12 +51,19 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/kr/pretty v0.2.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/ledgerwatch/log/v3 v3.7.0 // indirect
 	github.com/linxGnu/grocksdb v1.7.10 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
@@ -81,5 +95,5 @@ replace (
 	// use cometbft
 	github.com/tendermint/tendermint => github.com/cometbft/cometbft v0.34.27
 	// https://github.com/crypto-org-chain/tm-db/tree/release/v0.6.x
-	github.com/tendermint/tm-db => github.com/crypto-org-chain/tm-db v0.6.8-0.20230118040049-14dc6b00a5b3
+	github.com/tendermint/tm-db => github.com/crypto-org-chain/tm-db v0.6.8-0.20230412133356-5e16d347f5b9
 )