From 69a80154b6b24fca15f3562e2c4b312ee1092220 Mon Sep 17 00:00:00 2001
From: yihuang <huang@crypto.com>
Date: Tue, 9 Apr 2024 18:22:45 +0800
Subject: [PATCH] Problem: rocksdb wal format sometimes not backward compatible
 (backport #1379) (#1382)

Solution:
- flush the wal before quit the node, make the rocksdb upgrade smooth.

Update CHANGELOG.md



fix flush

Signed-off-by: yihuang <huang@crypto.com>
---
 .github/workflows/build.yml  |  2 +-
 CHANGELOG.md                 |  4 ++++
 app/app.go                   | 24 ++++++++++++++++--------
 versiondb/multistore.go      |  5 +++++
 versiondb/tsrocksdb/store.go | 10 ++++++++++
 versiondb/types.go           |  4 ++++
 6 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 06a471ff87..20f1250215 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -86,7 +86,7 @@ jobs:
             sed -i.bak "/$(echo $filename | sed 's/\//\\\//g')/d" coverage.txt
           done
         if: steps.changed-files.outputs.any_changed == 'true'
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.txt,./memiavl/coverage.txt,./store/coverage.txt,./versiondb/coverage.txt
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4357af2954..f7ec759f9b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 ## UNRELEASED
 
+### Improvements
+
+* (versiondb) [#1379](https://github.com/crypto-org-chain/cronos/pull/1379) Flush versiondb when graceful shutdown, make rocksdb upgrade smooth.
+
 ### Bug Fixes
 
 - [#1363](https://github.com/crypto-org-chain/cronos/pull/1363) Update ethermint to fix a panic on overflow and patch gasUsed in the RPC API.
diff --git a/app/app.go b/app/app.go
index 987c103492..1f5009b402 100644
--- a/app/app.go
+++ b/app/app.go
@@ -142,7 +142,6 @@ import (
 	// this line is used by starport scaffolding # stargate/app/moduleImport
 
 	memiavlstore "github.com/crypto-org-chain/cronos/store"
-	memiavlrootmulti "github.com/crypto-org-chain/cronos/store/rootmulti"
 	"github.com/crypto-org-chain/cronos/v2/x/cronos"
 	cronosclient "github.com/crypto-org-chain/cronos/v2/x/cronos/client"
 	cronoskeeper "github.com/crypto-org-chain/cronos/v2/x/cronos/keeper"
@@ -369,6 +368,8 @@ type App struct {
 
 	// module configurator
 	configurator module.Configurator
+
+	qms storetypes.MultiStore
 }
 
 // New returns a reference to an initialized chain.
@@ -859,10 +860,9 @@ func New(
 
 	// wire up the versiondb's `StreamingService` and `MultiStore`.
 	streamers := cast.ToStringSlice(appOpts.Get("store.streamers"))
-	var qms sdk.MultiStore
 	if slices.Contains(streamers, "versiondb") {
 		var err error
-		qms, err = app.setupVersionDB(homePath, keys, tkeys, memKeys)
+		app.qms, err = app.setupVersionDB(homePath, keys, tkeys, memKeys)
 		if err != nil {
 			panic(err)
 		}
@@ -899,8 +899,8 @@ func New(
 			tmos.Exit(err.Error())
 		}
 
-		if qms != nil {
-			v1 := qms.LatestVersion()
+		if app.qms != nil {
+			v1 := app.qms.LatestVersion()
 			v2 := app.LastBlockHeight()
 			if v1 > 0 && v1 < v2 {
 				// try to prevent gap being created in versiondb
@@ -1196,11 +1196,19 @@ func VerifyAddressFormat(bz []byte) error {
 
 // Close will be called in graceful shutdown in start cmd
 func (app *App) Close() error {
-	err := app.BaseApp.Close()
+	errs := []error{app.BaseApp.Close()}
+
+	// flush the versiondb
+	if closer, ok := app.qms.(io.Closer); ok {
+		errs = append(errs, closer.Close())
+	}
 
-	if cms, ok := app.CommitMultiStore().(*memiavlrootmulti.Store); ok {
-		return stderrors.Join(err, cms.Close())
+	// mainly to flush memiavl
+	if closer, ok := app.CommitMultiStore().(io.Closer); ok {
+		errs = append(errs, closer.Close())
 	}
 
+	err := stderrors.Join(errs...)
+	app.Logger().Info("Application gracefully shutdown", "error", err)
 	return err
 }
diff --git a/versiondb/multistore.go b/versiondb/multistore.go
index 878f6dd08e..8f61075dc7 100644
--- a/versiondb/multistore.go
+++ b/versiondb/multistore.go
@@ -145,3 +145,8 @@ func (s *MultiStore) LatestVersion() int64 {
 	}
 	return version
 }
+
+// Close will flush the versiondb
+func (s *MultiStore) Close() error {
+	return s.versionDB.Flush()
+}
diff --git a/versiondb/tsrocksdb/store.go b/versiondb/tsrocksdb/store.go
index b927ea36b9..be34c74632 100644
--- a/versiondb/tsrocksdb/store.go
+++ b/versiondb/tsrocksdb/store.go
@@ -205,6 +205,16 @@ func (s Store) Import(version int64, ch <-chan versiondb.ImportEntry) error {
 	return s.SetLatestVersion(version)
 }
 
+func (s Store) Flush() error {
+	opts := grocksdb.NewDefaultFlushOptions()
+	defer opts.Destroy()
+
+	return errors.Join(
+		s.db.Flush(opts),
+		s.db.FlushCF(s.cfHandle, opts),
+	)
+}
+
 func newTSReadOptions(version *int64) *grocksdb.ReadOptions {
 	var ver uint64
 	if version == nil {
diff --git a/versiondb/types.go b/versiondb/types.go
index 46bbe88e9e..6c69fbbf94 100644
--- a/versiondb/types.go
+++ b/versiondb/types.go
@@ -21,6 +21,10 @@ type VersionStore interface {
 
 	// Import the initial state of the store
 	Import(version int64, ch <-chan ImportEntry) error
+
+	// Flush wal logs, and make the changes persistent,
+	// mainly for rocksdb version upgrade, sometimes the wal format is not compatible.
+	Flush() error
 }
 
 type ImportEntry struct {