diff --git a/go.mod b/go.mod index 3679ed787d8a..fd6bea36e774 100644 --- a/go.mod +++ b/go.mod @@ -88,6 +88,14 @@ require ( k8s.io/utils v0.0.0-20230726121419-3b25d923346b ) +require ( + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/zondax/ledger-go v1.0.1 // indirect + rsc.io/tmplfunc v0.0.3 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect +) + require ( github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e // indirect github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec // indirect @@ -177,8 +185,7 @@ require ( github.com/sagikazarmark/locafero v0.9.0 // indirect github.com/sanity-io/litter v1.5.1 // indirect github.com/sirupsen/logrus v1.9.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.14.0 // indirect + github.com/spf13/afero v1.15.0 // indirect github.com/status-im/keycard-go v0.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect @@ -189,7 +196,6 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zondax/golem v0.27.0 // indirect github.com/zondax/hid v0.9.2 // indirect - github.com/zondax/ledger-go v1.0.1 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel/metric v1.37.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect @@ -202,9 +208,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - rsc.io/tmplfunc v0.0.3 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 27aaec848f0f..e96e597aa5c7 100644 --- a/go.sum +++ b/go.sum @@ -500,8 +500,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= -github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE= github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= diff --git a/tests/e2e/c/api.go b/tests/e2e/c/api.go index 54ce4393f904..7ef749f89ea4 100644 --- a/tests/e2e/c/api.go +++ b/tests/e2e/c/api.go @@ -4,6 +4,7 @@ package c import ( + "fmt" "math/big" "connectrpc.com/connect" @@ -19,6 +20,46 @@ import ( pb "github.com/ava-labs/avalanchego/connectproto/pb/proposervm/proposervmconnect" ) +func byAdvancingCChainHeight(tc *e2e.GinkgoTestContext, minBlocks int) { + require := require.New(tc) + + tc.By(fmt.Sprintf("advance the C-Chain height minimum of %d blocks", minBlocks), func() { + env := e2e.GetEnv(tc) + nodeURI := env.GetRandomNodeURI() + ethClient := e2e.NewEthClient(tc, nodeURI) + senderKey := env.PreFundedKey + senderEthAddress := senderKey.EthAddress() + recipientKey := e2e.NewPrivateKey(tc) + recipientEthAddress := recipientKey.EthAddress() + + for i := 0; i < minBlocks; i++ { + // Create and send a simple transaction to trigger block production + nonce, err := ethClient.AcceptedNonceAt(tc.DefaultContext(), senderEthAddress) + require.NoError(err) + gasPrice := e2e.SuggestGasPrice(tc, ethClient) + tx := types.NewTransaction( + nonce, + recipientEthAddress, + big.NewInt(1000000000000000), + e2e.DefaultGasLimit, + gasPrice, + nil, + ) + + // Sign transaction + cChainID, err := ethClient.ChainID(tc.DefaultContext()) + require.NoError(err) + signer := types.NewEIP155Signer(cChainID) + signedTx, err := types.SignTx(tx, signer, senderKey.ToECDSA()) + require.NoError(err) + + // Send the transaction and wait for receipt + receipt := e2e.SendEthTransaction(tc, ethClient, signedTx) + require.Equal(types.ReceiptStatusSuccessful, receipt.Status) + } + }) +} + var _ = e2e.DescribeCChain("[ProposerVM API]", ginkgo.Label("proposervm"), func() { tc := e2e.NewTestContext() require := require.New(tc) @@ -27,39 +68,7 @@ var _ = e2e.DescribeCChain("[ProposerVM API]", ginkgo.Label("proposervm"), func( env := e2e.GetEnv(tc) nodeURI := env.GetRandomNodeURI() - tc.By("advancing the C-chain height", func() { - ethClient := e2e.NewEthClient(tc, nodeURI) - senderKey := env.PreFundedKey - senderEthAddress := senderKey.EthAddress() - recipientKey := e2e.NewPrivateKey(tc) - recipientEthAddress := recipientKey.EthAddress() - - for i := 0; i < 3; i++ { - // Create and send a simple transaction to trigger block production - nonce, err := ethClient.AcceptedNonceAt(tc.DefaultContext(), senderEthAddress) - require.NoError(err) - gasPrice := e2e.SuggestGasPrice(tc, ethClient) - tx := types.NewTransaction( - nonce, - recipientEthAddress, - big.NewInt(1000000000000000), - e2e.DefaultGasLimit, - gasPrice, - nil, - ) - - // Sign transaction - cChainID, err := ethClient.ChainID(tc.DefaultContext()) - require.NoError(err) - signer := types.NewEIP155Signer(cChainID) - signedTx, err := types.SignTx(tx, signer, senderKey.ToECDSA()) - require.NoError(err) - - // Send the transaction and wait for receipt - receipt := e2e.SendEthTransaction(tc, ethClient, signedTx) - require.Equal(types.ReceiptStatusSuccessful, receipt.Status) - } - }) + byAdvancingCChainHeight(tc, 3) // Get the proper C-chain ID for routing keychain := env.NewKeychain() diff --git a/tests/e2e/c/fetch_blocks.go b/tests/e2e/c/fetch_blocks.go new file mode 100644 index 000000000000..1581d926a021 --- /dev/null +++ b/tests/e2e/c/fetch_blocks.go @@ -0,0 +1,79 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package c + +import ( + "os" + + "github.com/ava-labs/libevm/core/types" + "github.com/ava-labs/libevm/rlp" + "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/require" + + "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/tests/fixture/e2e" + "github.com/ava-labs/avalanchego/tests/reexecute/utils" +) + +var _ = e2e.DescribeCChain("[Fetch Blocks]", func() { + tc := e2e.NewTestContext() + require := require.New(tc) + + ginkgo.It("fetches created blocks", func() { + env := e2e.GetEnv(tc) + nodeURI := env.GetRandomNodeURI() + cChainNodeURI := nodeURI.URI + "/ext/bc/C/rpc" + + fetchBlocksDBDir, err := os.MkdirTemp("", "fetch-blocks-test") + require.NoError(err) + defer func() { + require.NoError(os.RemoveAll(fetchBlocksDBDir)) + }() + const ( + startBlock = 1 + endBlock = 10 + numBlocks = 10 + ) + byAdvancingCChainHeight(tc, numBlocks) + ginkgo.By("fetching blocks", func() { + require.NoError(utils.FetchBlocksToBlockDB( + tc.DefaultContext(), + tc.Log(), + fetchBlocksDBDir, + startBlock, + endBlock, + cChainNodeURI, + numBlocks, + )) + }) + ginkgo.By("checking blockDB contents", func() { + blockDB, err := utils.NewBlockDB(fetchBlocksDBDir) + require.NoError(err) + + defer func() { + require.NoError(blockDB.Close()) + }() + + blockIter := blockDB.NewIteratorFromHeight(startBlock) + + expectedBlock := uint64(startBlock) + for blockIter.Next() { + blockHeightKey := blockIter.Key() + blockBytes := blockIter.Value() + + require.Len(blockHeightKey, database.Uint64Size) + blockHeight, err := database.ParseUInt64(blockHeightKey) + require.NoError(err) + require.Equal(expectedBlock, blockHeight) + expectedBlock++ + + block := new(types.Block) + require.NoError(rlp.DecodeBytes(blockBytes, block)) + require.Equal(blockHeight, block.NumberU64()) + } + require.Equal(expectedBlock, uint64(endBlock)+1) + require.NoError(blockIter.Error()) + }) + }) +}) diff --git a/tests/reexecute/c/cli/cmd/fetch_blocks.go b/tests/reexecute/c/cli/cmd/fetch_blocks.go new file mode 100644 index 000000000000..5b859e678c6a --- /dev/null +++ b/tests/reexecute/c/cli/cmd/fetch_blocks.go @@ -0,0 +1,62 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package cmd + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/ava-labs/avalanchego/tests/reexecute/utils" +) + +const ( + dbDirKey = "db-dir" + startBlockKey = "start-block" + endBlockKey = "end-block" + rpcURLKey = "rpc-url" + concurrencyKey = "concurrency" +) + +// fetchBlocksCmd represents the fetchBlocks command +var fetchBlocksCmd = &cobra.Command{ + Use: "fetchBlocks", + Short: "Fetch blocks from the network and write to the specified database", + RunE: runFetchBlocks, +} + +func init() { + rootCmd.AddCommand(fetchBlocksCmd) + + fetchBlocksCmd.Flags().String(dbDirKey, "", "Database to store the fetched blocks") + fetchBlocksCmd.Flags().String(rpcURLKey, "http://localhost:9650/ext/bc/C/rpc", "Ethereum RPC URL to fetch blocks from") + fetchBlocksCmd.Flags().Uint64(startBlockKey, 0, "Block number to start fetching from (inclusive)") + fetchBlocksCmd.Flags().Uint64(endBlockKey, 0, "Block number to stop fetching at (inclusive)") + fetchBlocksCmd.Flags().Int(concurrencyKey, 1000, "Number of concurrent fetches to make") +} + +func runFetchBlocks(cmd *cobra.Command, _ []string) error { + dbDir, err := cmd.Flags().GetString(dbDirKey) + if err != nil { + return fmt.Errorf("failed to get db-dir flag: %w", err) + } + startBlock, err := cmd.Flags().GetUint64(startBlockKey) + if err != nil { + return fmt.Errorf("failed to get start-block flag: %w", err) + } + endBlock, err := cmd.Flags().GetUint64(endBlockKey) + if err != nil { + return fmt.Errorf("failed to get end-block flag: %w", err) + } + rpcURL, err := cmd.Flags().GetString(rpcURLKey) + if err != nil { + return fmt.Errorf("failed to get rpc-url flag: %w", err) + } + concurrency, err := cmd.Flags().GetInt(concurrencyKey) + if err != nil { + return fmt.Errorf("failed to get concurrency flag: %w", err) + } + return utils.FetchBlocksToBlockDB(context.Background(), cliLog, dbDir, startBlock, endBlock, rpcURL, concurrency) +} diff --git a/tests/reexecute/c/cli/cmd/root.go b/tests/reexecute/c/cli/cmd/root.go new file mode 100644 index 000000000000..43b24e96d009 --- /dev/null +++ b/tests/reexecute/c/cli/cmd/root.go @@ -0,0 +1,86 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/ava-labs/avalanchego/utils/logging" +) + +var ( + cfgFile string + logLevelKey = "log-level" + cliLog logging.Logger +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "argo", + Short: "Simple CLI tool to interact with re-execution", +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize(initConfig) + + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.cli.yaml)") + rootCmd.PersistentFlags().String(logLevelKey, logging.Info.String(), "Log level") + + rootCmd.PersistentPreRunE = func(_ *cobra.Command, _ []string) error { + logLevelStr, err := rootCmd.PersistentFlags().GetString(logLevelKey) + if err != nil { + return fmt.Errorf("failed to get %q flag: %w", logLevelKey, err) + } + logLevel, err := logging.ToLevel(logLevelStr) + if err != nil { + return fmt.Errorf("failed to parse log-level flag %q: %w", logLevelStr, err) + } + cliLog = logging.NewLogger( + "argo", + logging.NewWrappedCore( + logLevel, + os.Stdout, + logging.Colors.ConsoleEncoder(), + ), + ) + return nil + } +} + +// initConfig reads in config file and ENV variables if set. +func initConfig() { + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + // Find home directory. + home, err := os.UserHomeDir() + cobra.CheckErr(err) + + // Search config in home directory with name ".cli" (without extension). + viper.AddConfigPath(home) + viper.SetConfigType("yaml") + viper.SetConfigName(".cli") + } + + viper.AutomaticEnv() // read in environment variables that match + + // If a config file is found, read it in. + if err := viper.ReadInConfig(); err == nil { + fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) + } +} diff --git a/tests/reexecute/c/cli/main.go b/tests/reexecute/c/cli/main.go new file mode 100644 index 000000000000..1260591c491b --- /dev/null +++ b/tests/reexecute/c/cli/main.go @@ -0,0 +1,10 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package main + +import "github.com/ava-labs/avalanchego/tests/reexecute/c/cli/cmd" + +func main() { + cmd.Execute() +} diff --git a/tests/reexecute/c/vm_reexecute_test.go b/tests/reexecute/c/vm_reexecute_test.go index 9982da655e3d..9894a91af885 100644 --- a/tests/reexecute/c/vm_reexecute_test.go +++ b/tests/reexecute/c/vm_reexecute_test.go @@ -38,12 +38,12 @@ import ( "github.com/ava-labs/avalanchego/snow/validators/validatorstest" "github.com/ava-labs/avalanchego/tests" "github.com/ava-labs/avalanchego/tests/fixture/tmpnet" + "github.com/ava-labs/avalanchego/tests/reexecute/utils" "github.com/ava-labs/avalanchego/upgrade" "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/crypto/bls/signer/localsigner" "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/timer" - "github.com/ava-labs/avalanchego/utils/units" "github.com/ava-labs/avalanchego/vms/metervm" "github.com/ava-labs/avalanchego/vms/platformvm/warp" ) @@ -445,18 +445,18 @@ func createBlockChanFromLevelDB(tb testing.TB, sourceDir string, startBlock, end r := require.New(tb) ch := make(chan blockResult, chanSize) - db, err := leveldb.New(sourceDir, nil, logging.NoLog{}, prometheus.NewRegistry()) + blockDB, err := utils.NewBlockDB(sourceDir) if err != nil { - return nil, fmt.Errorf("failed to create leveldb database from %q: %w", sourceDir, err) + return nil, err } tb.Cleanup(func() { - r.NoError(db.Close()) + r.NoError(blockDB.Close()) }) go func() { defer close(ch) - iter := db.NewIteratorWithStart(blockKey(startBlock)) + iter := blockDB.NewIteratorFromHeight(startBlock) defer iter.Release() currentHeight := startBlock @@ -499,10 +499,6 @@ func createBlockChanFromLevelDB(tb testing.TB, sourceDir string, startBlock, end return ch, nil } -func blockKey(height uint64) []byte { - return binary.BigEndian.AppendUint64(nil, height) -} - func TestExportBlockRange(t *testing.T) { exportBlockRange(t, blockDirSrcArg, blockDirDstArg, startBlockArg, endBlockArg, chanSizeArg) } @@ -512,23 +508,13 @@ func exportBlockRange(tb testing.TB, blockDirSrc string, blockDirDst string, sta blockChan, err := createBlockChanFromLevelDB(tb, blockDirSrc, startBlock, endBlock, chanSize) r.NoError(err) - db, err := leveldb.New(blockDirDst, nil, logging.NoLog{}, prometheus.NewRegistry()) + blockDB, err := utils.NewBlockDB(blockDirDst) r.NoError(err) - tb.Cleanup(func() { - r.NoError(db.Close()) - }) + r.NoError(blockDB.Close()) - batch := db.NewBatch() for blkResult := range blockChan { - r.NoError(batch.Put(blockKey(blkResult.Height), blkResult.BlockBytes)) - - if batch.Size() > 10*units.MiB { - r.NoError(batch.Write()) - batch = db.NewBatch() - } + r.NoError(blockDB.WriteBlock(blkResult.Height, blkResult.BlockBytes)) } - - r.NoError(batch.Write()) } type consensusMetrics struct { diff --git a/tests/reexecute/utils/blockdb.go b/tests/reexecute/utils/blockdb.go new file mode 100644 index 000000000000..c2867a612781 --- /dev/null +++ b/tests/reexecute/utils/blockdb.go @@ -0,0 +1,54 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package utils + +import ( + "encoding/binary" + "fmt" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/ava-labs/avalanchego/database" + "github.com/ava-labs/avalanchego/database/leveldb" + "github.com/ava-labs/avalanchego/utils/logging" +) + +type BlockDB struct { + db database.Database +} + +func NewBlockDB(dbDir string) (*BlockDB, error) { + db, err := leveldb.New(dbDir, nil, logging.NoLog{}, prometheus.NewRegistry()) + if err != nil { + return nil, fmt.Errorf("failed to create leveldb block database from %q: %w", dbDir, err) + } + return &BlockDB{db: db}, nil +} + +func (b *BlockDB) WriteBlock(height uint64, bytes []byte) error { + if err := b.db.Put(blockKey(height), bytes); err != nil { + return fmt.Errorf("failed to write block at height %d: %w", height, err) + } + return nil +} + +func (b *BlockDB) ReadBlock(height uint64) ([]byte, error) { + bytes, err := b.db.Get(blockKey(height)) + if err != nil { + return nil, fmt.Errorf("failed to read block at height %d: %w", height, err) + } + return bytes, nil +} + +func (b *BlockDB) NewIteratorFromHeight(height uint64) database.Iterator { + return b.db.NewIteratorWithStartAndPrefix(blockKey(height), nil) +} + +func (b *BlockDB) Close() error { + return b.db.Close() +} + +func blockKey(height uint64) []byte { + return binary.BigEndian.AppendUint64(nil, height) +} diff --git a/tests/reexecute/utils/fetch_blocks.go b/tests/reexecute/utils/fetch_blocks.go new file mode 100644 index 000000000000..2b0687597479 --- /dev/null +++ b/tests/reexecute/utils/fetch_blocks.go @@ -0,0 +1,77 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package utils + +import ( + "context" + "fmt" + "math/big" + + "github.com/ava-labs/libevm/rlp" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + "github.com/ava-labs/avalanchego/utils/logging" + + ethclient "github.com/ava-labs/coreth/plugin/evm/customethclient" +) + +func FetchBlocksToBlockDB(ctx context.Context, log logging.Logger, dbDir string, startBlock, endBlock uint64, rpcURL string, concurrency int) error { + client, err := ethclient.Dial(rpcURL) + if err != nil { + return fmt.Errorf("failed to connect to RPC URL %s: %w", rpcURL, err) + } + + blockDB, err := NewBlockDB(dbDir) + if err != nil { + return err + } + defer blockDB.Close() + + log.Info("Fetching blocks", + zap.Uint64("startBlock", startBlock), + zap.Uint64("endBlock", endBlock), + zap.String("rpcURL", rpcURL), + zap.String("dbDir", dbDir), + ) + + blocksCh := make(chan uint64, concurrency) + go func() { + defer close(blocksCh) + for i := startBlock; i <= endBlock; i++ { + select { + case blocksCh <- i: + case <-ctx.Done(): + return + } + } + }() + + eg, ctx := errgroup.WithContext(ctx) + for i := 0; i < concurrency; i++ { + eg.Go(func() error { + for blockNum := range blocksCh { + log.Debug("Fetching block", zap.Uint64("blockNumber", blockNum)) + block, err := client.BlockByNumber(ctx, new(big.Int).SetUint64(blockNum)) + if err != nil { + return fmt.Errorf("failed to fetch block %d: %w", blockNum, err) + } + log.Debug("Fetched block", + zap.Uint64("blockNumber", blockNum), + zap.Stringer("blockHash", block.Hash()), + ) + + blockBytes, err := rlp.EncodeToBytes(block) + if err != nil { + return fmt.Errorf("failed to encode block %d: %w", blockNum, err) + } + if err := blockDB.WriteBlock(blockNum, blockBytes); err != nil { + return err + } + } + return nil + }) + } + return eg.Wait() +}