feat: big genesis file (#576)

* feat: big genesis file

* chore: remove unused import

* test: fix lint test

* test: add test

* feat: decrease db block chunk size

* test: commend out test

* Revert "feat: big genesis file"

This reverts commit 7141d54.

* feat: restore codes

* Revert "feat: decrease db block chunk size"

This reverts commit 100d340.

* Revert "feat: big genesis file"

* feat: fix rocksdb error

* chore: remove unnecessary err check

* test: change NewDB to NewGoLevelDB

node/node.go

@@ -168,12 +168,12 @@ type fastSyncReactor interface {
 // WARNING: using any name from the below list of the existing reactors will
 // result in replacing it with the custom one.
 //
-//  - MEMPOOL
-//  - BLOCKCHAIN
-//  - CONSENSUS
-//  - EVIDENCE
-//  - PEX
-//  - STATESYNC
+//   - MEMPOOL
+//   - BLOCKCHAIN
+//   - CONSENSUS
+//   - EVIDENCE
+//   - PEX
+//   - STATESYNC
 func CustomReactors(reactors map[string]p2p.Reactor) Option {
 	return func(n *Node) {
 		for name, reactor := range reactors {
@@ -1423,13 +1423,23 @@ func LoadStateFromDBOrGenesisDocProvider(
 
 // panics if failed to unmarshal bytes
 func loadGenesisDoc(db dbm.DB) (*types.GenesisDoc, error) {
-	b, err := db.Get(genesisDocKey)
+	var b []byte
+	iter, err := db.Iterator(append(genesisDocKey, byte(0)), append(genesisDocKey, byte(255)))
 	if err != nil {
-		panic(err)
+		return nil, err
+	}
+
+	for ; iter.Valid(); iter.Next() {
+		b = append(b, iter.Value()...)
+	}
+	if err = iter.Close(); err != nil {
+		return nil, err
 	}
+
 	if len(b) == 0 {
 		return nil, errors.New("genesis doc not found")
 	}
+
 	var genDoc *types.GenesisDoc
 	err = tmjson.Unmarshal(b, &genDoc)
 	if err != nil {
@@ -1444,7 +1454,27 @@ func saveGenesisDoc(db dbm.DB, genDoc *types.GenesisDoc) error {
 	if err != nil {
 		return fmt.Errorf("failed to save genesis doc due to marshaling error: %w", err)
 	}
-	if err := db.SetSync(genesisDocKey, b); err != nil {
+
+	blockSize := 100000000 // 100mb
+	blocks := make([][]byte, 0)
+	for i := 0; i < len(b); i += blockSize {
+		end := i + blockSize
+		if end > len(b) {
+			end = len(b)
+		}
+		blocks = append(blocks, b[i:end])
+	}
+
+	batch := db.NewBatch()
+	for i, block := range blocks {
+		k := append(genesisDocKey, byte(i))
+		err = batch.Set(k, block)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err = batch.WriteSync(); err != nil {
 		return err
 	}
 

node/node_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"strings"
 	"syscall"
 	"testing"
 	"time"
@@ -609,6 +610,23 @@ func TestNodeInvalidNodeInfoCustomReactors(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestSaveAndLoadBigGensisFile(t *testing.T) {
+	stateDB, err := dbm.NewGoLevelDB("state", os.TempDir())
+	require.NoError(t, err)
+	config := cfg.ResetTestRoot("node_big_genesis_test")
+	defer os.RemoveAll(config.RootDir)
+	n, err := DefaultNewNode(config, log.TestingLogger())
+	require.NoError(t, err)
+	newChainID := strings.Repeat("a", 200000000) // about 200MB
+	n.genesisDoc.ChainID = newChainID
+	err = saveGenesisDoc(stateDB, n.genesisDoc)
+	require.NoError(t, err)
+	g, err := loadGenesisDoc(stateDB)
+	require.NoError(t, err)
+	require.Equal(t, newChainID, g.ChainID)
+	stateDB.Close()
+}
+
 func NewInvalidNode(config *cfg.Config,
 	privValidator types.PrivValidator,
 	nodeKey *p2p.NodeKey,