feat!: Add hooks to allow app modules to add things to state-sync (#1…

…0961)

## Description

Closes: #7340

- Support registering multiple snapshotters in snapshot manager.
- Append the extension snapshotters to existing snapshot stream.

~TODO: testing.~
- existing tests are fixed

---

### Author Checklist

*All items are required. Please add a note to the item if the item is not applicable and
please add links to any relevant follow up issues.*

I have...

- [ ] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] added `!` to the type prefix if API or client breaking change
- [ ] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting))
- [ ] provided a link to the relevant issue or specification
- [ ] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules)
- [ ] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing)
- [ ] added a changelog entry to `CHANGELOG.md`
- [ ] included comments for [documenting Go code](https://blog.golang.org/godoc)
- [ ] updated the relevant documentation or specification
- [ ] reviewed "Files changed" and left comments if necessary
- [ ] confirmed all CI checks have passed

### Reviewers Checklist

*All items are required. Please add a note if the item is not applicable and please add
your handle next to the items reviewed if you only reviewed selected items.*

I have...

- [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed
- [ ] reviewed state machine logic
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)

(cherry picked from commit 7e18e9f)

# Conflicts:
#	api/cosmos/base/snapshots/v1beta1/snapshot.pulsar.go
#	server/mock/store.go
#	snapshots/helpers_test.go
#	snapshots/manager.go
#	store/rootmulti/store_test.go
#	store/v2/multi/store.go

baseapp/baseapp.go

@@ -273,6 +273,22 @@ func DefaultStoreLoader(ms sdk.CommitMultiStore) error {
 	return ms.LoadLatestVersion()
 }
 
+// CommitMultiStore returns the root multi-store.
+// App constructor can use this to access the `cms`.
+// UNSAFE: only safe to use during app initialization.
+func (app *BaseApp) CommitMultiStore() sdk.CommitMultiStore {
+	if app.sealed {
+		panic("cannot call CommitMultiStore() after baseapp is sealed")
+	}
+	return app.cms
+}
+
+// SnapshotManager returns the snapshot manager.
+// application use this to register extra extension snapshotters.
+func (app *BaseApp) SnapshotManager() *snapshots.Manager {
+	return app.snapshotManager
+}
+
 // LoadVersion loads the BaseApp application version. It will panic if called
 // more than once on a running baseapp.
 func (app *BaseApp) LoadVersion(version int64) error {

baseapp/options.go

@@ -217,7 +217,7 @@ func (app *BaseApp) SetSnapshotStore(snapshotStore *snapshots.Store) {
 		app.snapshotManager = nil
 		return
 	}
-	app.snapshotManager = snapshots.NewManager(snapshotStore, app.cms)
+	app.snapshotManager = snapshots.NewManager(snapshotStore, app.cms, nil)
 }
 
 // SetSnapshotInterval sets the snapshot interval.

proto/cosmos/base/snapshots/v1beta1/snapshot.proto

@@ -17,4 +17,41 @@ message Snapshot {
 // Metadata contains SDK-specific snapshot metadata.
 message Metadata {
   repeated bytes chunk_hashes = 1; // SHA-256 chunk hashes
-}
+}
+
+// SnapshotItem is an item contained in a rootmulti.Store snapshot.
+message SnapshotItem {
+  // item is the specific type of snapshot item.
+  oneof item {
+    SnapshotStoreItem        store             = 1;
+    SnapshotIAVLItem         iavl              = 2 [(gogoproto.customname) = "IAVL"];
+    SnapshotExtensionMeta    extension         = 3;
+    SnapshotExtensionPayload extension_payload = 4;
+  }
+}
+
+// SnapshotStoreItem contains metadata about a snapshotted store.
+message SnapshotStoreItem {
+  string name = 1;
+}
+
+// SnapshotIAVLItem is an exported IAVL node.
+message SnapshotIAVLItem {
+  bytes key     = 1;
+  bytes value   = 2;
+  // version is block height
+  int64 version = 3;
+  // height is depth of the tree.
+  int32 height  = 4;
+}
+
+// SnapshotExtensionMeta contains metadata about an external snapshotter.
+message SnapshotExtensionMeta {
+  string name   = 1;
+  uint32 format = 2;
+}
+
+// SnapshotExtensionPayload contains payloads of an external snapshotter.
+message SnapshotExtensionPayload {
+  bytes payload = 1;
+}

server/mock/store.go

@@ -3,9 +3,15 @@ package mock
 import (
 	"io"
 
+	protoio "github.com/gogo/protobuf/io"
 	dbm "github.com/tendermint/tm-db"
 
+<<<<<<< HEAD
 	store "github.com/cosmos/cosmos-sdk/store/types"
+=======
+	snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
+	storetypes "github.com/cosmos/cosmos-sdk/store/types"
+>>>>>>> 7e18e9f1b (feat!: Add hooks to allow app modules to add things to state-sync (#10961))
 	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
@@ -122,13 +128,13 @@ func (ms multiStore) SetInitialVersion(version int64) error {
 	panic("not implemented")
 }
 
-func (ms multiStore) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, error) {
+func (ms multiStore) Snapshot(height uint64, protoWriter protoio.Writer) error {
 	panic("not implemented")
 }
 
 func (ms multiStore) Restore(
-	height uint64, format uint32, chunks <-chan io.ReadCloser, ready chan<- struct{},
-) error {
+	height uint64, format uint32, protoReader protoio.Reader,
+) (snapshottypes.SnapshotItem, error) {
 	panic("not implemented")
 }
 

snapshots/helpers_test.go

@@ -1,7 +1,9 @@
 package snapshots_test
 
 import (
+	"bufio"
 	"bytes"
+	"compress/zlib"
 	"crypto/sha256"
 	"errors"
 	"io"
@@ -10,11 +12,14 @@ import (
 	"testing"
 	"time"
 
+	protoio "github.com/gogo/protobuf/io"
 	"github.com/stretchr/testify/require"
 	db "github.com/tendermint/tm-db"
 
 	"github.com/cosmos/cosmos-sdk/snapshots"
 	"github.com/cosmos/cosmos-sdk/snapshots/types"
+	snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
+	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 )
 
 func checksums(slice [][]byte) [][]byte {
@@ -57,35 +62,88 @@ func readChunks(chunks <-chan io.ReadCloser) [][]byte {
 	return bodies
 }
 
+// snapshotItems serialize a array of bytes as SnapshotItem_ExtensionPayload, and return the chunks.
+func snapshotItems(items [][]byte) [][]byte {
+	// copy the same parameters from the code
+	snapshotChunkSize := uint64(10e6)
+	snapshotBufferSize := int(snapshotChunkSize)
+
+	ch := make(chan io.ReadCloser)
+	go func() {
+		chunkWriter := snapshots.NewChunkWriter(ch, snapshotChunkSize)
+		bufWriter := bufio.NewWriterSize(chunkWriter, snapshotBufferSize)
+		zWriter, _ := zlib.NewWriterLevel(bufWriter, 7)
+		protoWriter := protoio.NewDelimitedWriter(zWriter)
+		for _, item := range items {
+			types.WriteExtensionItem(protoWriter, item)
+		}
+		protoWriter.Close()
+		zWriter.Close()
+		bufWriter.Flush()
+		chunkWriter.Close()
+	}()
+
+	var chunks [][]byte
+	for chunkBody := range ch {
+		chunk, err := io.ReadAll(chunkBody)
+		if err != nil {
+			panic(err)
+		}
+		chunks = append(chunks, chunk)
+	}
+	return chunks
+}
+
 type mockSnapshotter struct {
-	chunks [][]byte
+	items [][]byte
 }
 
 func (m *mockSnapshotter) Restore(
-	height uint64, format uint32, chunks <-chan io.ReadCloser, ready chan<- struct{},
-) error {
+	height uint64, format uint32, protoReader protoio.Reader,
+) (snapshottypes.SnapshotItem, error) {
 	if format == 0 {
-		return types.ErrUnknownFormat
+		return snapshottypes.SnapshotItem{}, types.ErrUnknownFormat
 	}
-	if m.chunks != nil {
-		return errors.New("already has contents")
-	}
-	if ready != nil {
-		close(ready)
+	if m.items != nil {
+		return snapshottypes.SnapshotItem{}, errors.New("already has contents")
 	}
 
+<<<<<<< HEAD
 	m.chunks = [][]byte{}
 	for reader := range chunks {
 		chunk, err := ioutil.ReadAll(reader)
 		if err != nil {
-			return err
+=======
+	m.items = [][]byte{}
+	for {
+		item := &snapshottypes.SnapshotItem{}
+		err := protoReader.ReadMsg(item)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(err, "invalid protobuf message")
+		}
+		payload := item.GetExtensionPayload()
+		if payload == nil {
+			return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(err, "invalid protobuf message")
 		}
-		m.chunks = append(m.chunks, chunk)
+		m.items = append(m.items, payload.Payload)
 	}
 
+	return snapshottypes.SnapshotItem{}, nil
+}
+
+func (m *mockSnapshotter) Snapshot(height uint64, protoWriter protoio.Writer) error {
+	for _, item := range m.items {
+		if err := types.WriteExtensionItem(protoWriter, item); err != nil {
+>>>>>>> 7e18e9f1b (feat!: Add hooks to allow app modules to add things to state-sync (#10961))
+			return err
+		}
+	}
 	return nil
 }
 
+<<<<<<< HEAD
 func (m *mockSnapshotter) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, error) {
 	if format == 0 {
 		return nil, types.ErrUnknownFormat
@@ -96,6 +154,13 @@ func (m *mockSnapshotter) Snapshot(height uint64, format uint32) (<-chan io.Read
 	}
 	close(ch)
 	return ch, nil
+=======
+func (m *mockSnapshotter) SnapshotFormat() uint32 {
+	return 1
+}
+func (m *mockSnapshotter) SupportedFormats() []uint32 {
+	return []uint32{1}
+>>>>>>> 7e18e9f1b (feat!: Add hooks to allow app modules to add things to state-sync (#10961))
 }
 
 // setupBusyManager creates a manager with an empty store that is busy creating a snapshot at height 1.
@@ -111,7 +176,7 @@ func setupBusyManager(t *testing.T) *snapshots.Manager {
 	store, err := snapshots.NewStore(db.NewMemDB(), tempdir)
 	require.NoError(t, err)
 	hung := newHungSnapshotter()
-	mgr := snapshots.NewManager(store, hung)
+	mgr := snapshots.NewManager(store, hung, nil)
 
 	go func() {
 		_, err := mgr.Create(1)
@@ -138,15 +203,19 @@ func (m *hungSnapshotter) Close() {
 	close(m.ch)
 }
 
-func (m *hungSnapshotter) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, error) {
+func (m *hungSnapshotter) Snapshot(height uint64, protoWriter protoio.Writer) error {
 	<-m.ch
+<<<<<<< HEAD
 	ch := make(chan io.ReadCloser, 1)
 	ch <- ioutil.NopCloser(bytes.NewReader([]byte{}))
 	return ch, nil
+=======
+	return nil
+>>>>>>> 7e18e9f1b (feat!: Add hooks to allow app modules to add things to state-sync (#10961))
 }
 
 func (m *hungSnapshotter) Restore(
-	height uint64, format uint32, chunks <-chan io.ReadCloser, ready chan<- struct{},
-) error {
+	height uint64, format uint32, protoReader protoio.Reader,
+) (snapshottypes.SnapshotItem, error) {
 	panic("not implemented")
 }