fix(snapshots): raise the per snapshot item limit (#304)

Increases the snapshot per item size limit when restoring from state-sync

snapshots/manager.go

@@ -21,7 +21,10 @@ const (
 
 	chunkBufferSize = 4
 
-	snapshotMaxItemSize = int(64e6) // SDK has no key/value size limit, so we set an arbitrary limit
+	// snapshotMaxItemSize limits the size of both KVStore entries and snapshot
+	// extension payloads during a state-sync restore.
+	// Unexported so copied in manager_test.go for testing
+	snapshotMaxItemSize = int(512e6)
 )
 
 // operation represents a Manager operation. Only one operation can be in progress at a time.

snapshots/manager_test.go

@@ -2,6 +2,7 @@ package snapshots_test
 
 import (
 	"errors"
+	"io"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -218,3 +219,89 @@ func TestManager_Restore(t *testing.T) {
 	})
 	require.NoError(t, err)
 }
+
+const snapshotMaxItemSize = int(512e6) // Copied from github.com/cosmos/cosmos-sdk/snapshots
+
+func TestManager_RestoreLargeItem(t *testing.T) {
+	store := setupStore(t)
+	target := &mockSnapshotter{}
+	extSnapshotter := newExtSnapshotter(0)
+	manager := snapshots.NewManager(store, target)
+	err := manager.RegisterExtensions(extSnapshotter)
+	require.NoError(t, err)
+
+	largeItem := make([]byte, snapshotMaxItemSize)
+
+	// The protobuf wrapper introduces extra bytes
+	adjustedSize := 2*snapshotMaxItemSize - (&types.SnapshotItem{
+		Item: &types.SnapshotItem_ExtensionPayload{
+			ExtensionPayload: &types.SnapshotExtensionPayload{
+				Payload: largeItem,
+			},
+		},
+	}).Size()
+	largeItem = largeItem[:adjustedSize]
+	expectItems := [][]byte{largeItem}
+
+	chunks := snapshotItems(expectItems, newExtSnapshotter(1))
+
+	// Starting a restore works
+	err = manager.Restore(types.Snapshot{
+		Height:   3,
+		Format:   2,
+		Hash:     []byte{1, 2, 3},
+		Chunks:   1,
+		Metadata: types.Metadata{ChunkHashes: checksums(chunks)},
+	})
+	require.NoError(t, err)
+
+	// Feeding the chunks should work
+	for i, chunk := range chunks {
+		done, err := manager.RestoreChunk(chunk)
+		require.NoError(t, err)
+		if i == len(chunks)-1 {
+			assert.True(t, done)
+		} else {
+			assert.False(t, done)
+		}
+	}
+
+	assert.Equal(t, expectItems, target.items)
+	assert.Equal(t, 1, len(extSnapshotter.state))
+}
+
+func TestManager_CannotRestoreTooLargeItem(t *testing.T) {
+	store := setupStore(t)
+	target := &mockSnapshotter{}
+	extSnapshotter := newExtSnapshotter(0)
+	manager := snapshots.NewManager(store, target)
+	err := manager.RegisterExtensions(extSnapshotter)
+	require.NoError(t, err)
+
+	// The protobuf wrapper introduces extra bytes
+	largeItem := make([]byte, snapshotMaxItemSize)
+	expectItems := [][]byte{largeItem}
+
+	chunks := snapshotItems(expectItems, newExtSnapshotter(1))
+
+	// Starting a restore works
+	err = manager.Restore(types.Snapshot{
+		Height:   3,
+		Format:   2,
+		Hash:     []byte{1, 2, 3},
+		Chunks:   1,
+		Metadata: types.Metadata{ChunkHashes: checksums(chunks)},
+	})
+	require.NoError(t, err)
+
+	// Feeding the chunks fails
+	for _, chunk := range chunks {
+		_, err = manager.RestoreChunk(chunk)
+
+		if err != nil {
+			break
+		}
+	}
+	require.Error(t, err)
+	require.True(t, errors.Is(err, io.ErrShortBuffer))
+}