Caplin: Added sync from scratch and resuming node without checkpoint sync

cl/clparams/config.go

@@ -36,11 +36,16 @@ import (
 	"github.com/erigontech/erigon/cl/utils"
 )
 
+var LatestStateFileName = "latest.ssz_snappy"
+
 type CaplinConfig struct {
 	Backfilling         bool
 	BlobBackfilling     bool
 	BlobPruningDisabled bool
 	Archive             bool
+	NetworkId           NetworkType
+	// DisableCheckpointSync is optional and is used to disable checkpoint sync used by default in the node
+	DisabledCheckpointSync bool
 	// CaplinMeVRelayUrl is optional and is used to connect to the external builder service.
 	// If it's set, the node will start in builder mode
 	MevRelayUrl string

cl/cltypes/solid/uint64slice_byte.go

@@ -141,10 +141,10 @@ func (arr *byteBasedUint64Slice) Append(v uint64) {
 	}
 	offset := arr.l * 8
 	binary.LittleEndian.PutUint64(arr.u[offset:offset+8], v)
-	arr.l++
 	if arr.MerkleTree != nil {
 		arr.MerkleTree.MarkLeafAsDirty(arr.l / 4)
 	}
+	arr.l++
 }
 
 // Get returns the element at the given index.

cl/phase1/core/checkpoint_sync/checkpoint_sync_test.go

@@ -0,0 +1,89 @@
+package checkpoint_sync
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/erigontech/erigon/cl/antiquary/tests"
+	"github.com/erigontech/erigon/cl/clparams"
+	"github.com/erigontech/erigon/cl/cltypes"
+	"github.com/erigontech/erigon/cl/utils"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemoteCheckpointSync(t *testing.T) {
+	_, st, _ := tests.GetPhase0Random()
+	rec := false
+	// Create a mock HTTP server
+	mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		enc, err := st.EncodeSSZ(nil)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("could not encode state: %s", err), http.StatusInternalServerError)
+			return
+		}
+		w.Write(enc)
+		rec = true
+	}))
+	defer mockServer.Close()
+
+	clparams.ConfigurableCheckpointsURLs = []string{mockServer.URL}
+	syncer := NewRemoteCheckpointSync(&clparams.MainnetBeaconConfig, clparams.MainnetNetwork)
+	state, err := syncer.GetLatestBeaconState(context.Background())
+	assert.True(t, rec)
+	require.NoError(t, err)
+	require.NotNil(t, state)
+	// Compare the roots of the states
+	haveRoot, err := st.HashSSZ()
+	require.NoError(t, err)
+	wantRoot, err := state.HashSSZ()
+	require.NoError(t, err)
+
+	assert.Equal(t, haveRoot, wantRoot)
+}
+
+func TestLocalCheckpointSyncFromFile(t *testing.T) {
+	_, st, _ := tests.GetPhase0Random()
+	f := afero.NewMemMapFs()
+	enc, err := st.EncodeSSZ(nil)
+	enc = utils.CompressSnappy(enc)
+	require.NoError(t, err)
+	require.NoError(t, afero.WriteFile(f, clparams.LatestStateFileName, enc, 0644))
+
+	genesisState, err := st.Copy()
+	require.NoError(t, err)
+	genesisState.AddEth1DataVote(cltypes.NewEth1Data()) // Add some data to the genesis state so that it is different from the state read from the file
+
+	syncer := NewLocalCheckpointSyncer(genesisState, f)
+	state, err := syncer.GetLatestBeaconState(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, state)
+	// Compare the roots of the states
+	haveRoot, err := st.HashSSZ()
+	require.NoError(t, err)
+	wantRoot, err := state.HashSSZ()
+	require.NoError(t, err)
+
+	assert.Equal(t, haveRoot, wantRoot)
+}
+
+func TestLocalCheckpointSyncFromGenesis(t *testing.T) {
+	_, st, _ := tests.GetPhase0Random()
+	f := afero.NewMemMapFs()
+
+	syncer := NewLocalCheckpointSyncer(st, f)
+	state, err := syncer.GetLatestBeaconState(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, state)
+	// Compare the roots of the states
+	haveRoot, err := st.HashSSZ()
+	require.NoError(t, err)
+	wantRoot, err := state.HashSSZ()
+	require.NoError(t, err)
+
+	assert.Equal(t, haveRoot, wantRoot)
+}

cl/phase1/core/checkpoint_sync/interface.go

@@ -0,0 +1,11 @@
+package checkpoint_sync
+
+import (
+	"context"
+
+	"github.com/erigontech/erigon/cl/phase1/core/state"
+)
+
+type CheckpointSyncer interface {
+	GetLatestBeaconState(ctx context.Context) (*state.CachingBeaconState, error)
+}

cl/phase1/core/checkpoint_sync/local_checkpoint_syncer.go

@@ -0,0 +1,50 @@
+package checkpoint_sync
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/erigontech/erigon-lib/log/v3"
+	"github.com/erigontech/erigon/cl/clparams"
+	"github.com/erigontech/erigon/cl/phase1/core/state"
+	"github.com/erigontech/erigon/cl/utils"
+	"github.com/spf13/afero"
+)
+
+type LocalCheckpointSyncer struct {
+	genesisState *state.CachingBeaconState
+	dir          afero.Fs
+}
+
+// The local checkpoint syncer, loads a checkpoint from the local disk or uses the genesis state.
+func NewLocalCheckpointSyncer(genesisState *state.CachingBeaconState, dir afero.Fs) CheckpointSyncer {
+	return &LocalCheckpointSyncer{
+		genesisState: genesisState,
+		dir:          dir,
+	}
+
+}
+
+func (l *LocalCheckpointSyncer) GetLatestBeaconState(ctx context.Context) (*state.CachingBeaconState, error) {
+	// Open file {latestStateSubDir}/{fileName}
+	snappyEncoded, err := afero.ReadFile(l.dir, clparams.LatestStateFileName)
+	if err != nil {
+		log.Warn("Could not read local state, starting sync from genesis.")
+		return l.genesisState.Copy()
+	}
+	decompressedSnappy, err := utils.DecompressSnappy(snappyEncoded)
+	if err != nil {
+		return nil, fmt.Errorf("local state is corrupt: %s", err)
+	}
+
+	beaconCfg := l.genesisState.BeaconConfig()
+	bs := state.New(beaconCfg)
+	slot, err := extractSlotFromSerializedBeaconState(decompressedSnappy)
+	if err != nil {
+		return nil, fmt.Errorf("could not deserialize state slot: %s", err)
+	}
+	if err := bs.DecodeSSZ(decompressedSnappy, int(beaconCfg.GetCurrentStateVersion(slot/beaconCfg.SlotsPerEpoch))); err != nil {
+		return nil, fmt.Errorf("could not deserialize state: %s", err)
+	}
+	return bs, nil
+}