Shuffle tests revisited

beacon-chain/blockchain/block_processing_test.go

@@ -510,7 +510,7 @@ func TestReceiveBlock_RemovesPendingDeposits(t *testing.T) {
 	}
 
 	pendingDeposits := []*pb.Deposit{
-		createPreChainStartDeposit([]byte{'F'}, beaconState.DepositIndex),
+		createPreChainStartDeposit([]byte{'F'}),
 	}
 	pendingDepositsData := make([][]byte, len(pendingDeposits))
 	for i, pd := range pendingDeposits {

beacon-chain/blockchain/service_test.go

@@ -151,7 +151,7 @@ func (mb *mockBroadcaster) Broadcast(_ context.Context, _ proto.Message) {
 
 var _ = p2p.Broadcaster(&mockBroadcaster{})
 
-func createPreChainStartDeposit(pk []byte, index uint64) *pb.Deposit {
+func createPreChainStartDeposit(pk []byte) *pb.Deposit {
 	balance := params.BeaconConfig().MaxDepositAmount
 	depositData := &pb.DepositData{Pubkey: pk, Amount: balance}
 

beacon-chain/chaintest/BUILD.bazel

@@ -8,7 +8,6 @@ go_library(
     deps = [
         "//beacon-chain/chaintest/backend:go_default_library",
         "//shared/featureconfig:go_default_library",
-        "//shared/params:go_default_library",
         "@com_github_go_yaml_yaml//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",

beacon-chain/chaintest/backend/BUILD.bazel

@@ -5,7 +5,6 @@ go_library(
     srcs = [
         "fork_choice_test_format.go",
         "helpers.go",
-        "shuffle_test_format.go",
         "simulated_backend.go",
         "state_test_format.go",
     ],
@@ -17,15 +16,13 @@ go_library(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/state:go_default_library",
         "//beacon-chain/db:go_default_library",
-        "//beacon-chain/utils:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/blockutil:go_default_library",
         "//shared/bls:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
         "//shared/trieutil:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],

beacon-chain/chaintest/backend/simulated_backend.go

@@ -6,16 +6,13 @@ package backend
 import (
 	"context"
 	"fmt"
-	"reflect"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/blockchain"
 	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	"github.com/prysmaticlabs/prysm/beacon-chain/utils"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bls"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
@@ -173,30 +170,6 @@ func (sb *SimulatedBackend) RunForkChoiceTest(testCase *ForkChoiceTestCase) erro
 	return nil
 }
 
-// RunShuffleTest uses validator set specified from a YAML file, runs the validator shuffle
-// algorithm, then compare the output with the expected output from the YAML file.
-func (sb *SimulatedBackend) RunShuffleTest(testCase *ShuffleTestCase) error {
-	defer db.TeardownDB(sb.beaconDB)
-	seed := common.HexToHash(testCase.Seed)
-	testIndices := make([]uint64, testCase.Count, testCase.Count)
-	for i := uint64(0); i < testCase.Count; i++ {
-		testIndices[i] = i
-	}
-	shuffledList := make([]uint64, testCase.Count)
-	for i := uint64(0); i < testCase.Count; i++ {
-		si, err := utils.ShuffledIndex(i, testCase.Count, seed)
-		if err != nil {
-			log.Error(err)
-			return err
-		}
-		shuffledList[i] = si
-	}
-	if !reflect.DeepEqual(shuffledList, testCase.Shuffled) {
-		return fmt.Errorf("shuffle result error: expected %v, actual %v", testCase.Shuffled, shuffledList)
-	}
-	return nil
-}
-
 // RunStateTransitionTest advances a beacon chain state transition an N amount of
 // slots from a genesis state, with a block being processed at every iteration
 // of the state transition function.

beacon-chain/chaintest/main.go

@@ -1,19 +1,15 @@
 package main
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"path"
-	"path/filepath"
-	"reflect"
 	"time"
 
 	"github.com/go-yaml/yaml"
 	"github.com/prysmaticlabs/prysm/beacon-chain/chaintest/backend"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/params"
 	log "github.com/sirupsen/logrus"
 	prefixed "github.com/x-cray/logrus-prefixed-formatter"
 )
@@ -24,64 +20,46 @@ func init() {
 	})
 }
 
-func readTestsFromYaml(yamlDir string) (tests []interface{}, configs map[string]interface{}, err error) {
+func readTestsFromYaml(yamlDir string) (tests []interface{}, err error) {
 	const forkChoiceTestsFolderName = "fork-choice-tests"
-	const shuffleTestsFolderName = "shuffle-tests"
 	const stateTestsFolderName = "state-tests"
-	const configFilesFolderName = "config"
-	configs = make(map[string]interface{})
 	dirs, err := ioutil.ReadDir(yamlDir)
 	if err != nil {
-		return nil, nil, fmt.Errorf("could not read YAML tests directory: %v", err)
+		return nil, fmt.Errorf("could not read YAML tests directory: %v", err)
 	}
 	for _, dir := range dirs {
 		files, err := ioutil.ReadDir(path.Join(yamlDir, dir.Name()))
 		if err != nil {
-			return nil, nil, fmt.Errorf("could not read YAML tests directory: %v", err)
+			return nil, fmt.Errorf("could not read YAML tests directory: %v", err)
 		}
 		for _, file := range files {
 			filePath := path.Join(yamlDir, dir.Name(), file.Name())
 			// #nosec G304
 			data, err := ioutil.ReadFile(filePath)
 			if err != nil {
-				return nil, nil, fmt.Errorf("could not read YAML file: %v", err)
+				return nil, fmt.Errorf("could not read YAML file: %v", err)
 			}
 			switch dir.Name() {
 			case forkChoiceTestsFolderName:
 				decoded := &backend.ForkChoiceTest{}
 				if err := yaml.Unmarshal(data, decoded); err != nil {
-					return nil, nil, fmt.Errorf("could not unmarshal YAML file into test struct: %v", err)
-				}
-				tests = append(tests, decoded)
-			case shuffleTestsFolderName:
-				decoded := &backend.ShuffleTest{}
-				if err := yaml.Unmarshal(data, decoded); err != nil {
-					return nil, nil, fmt.Errorf("could not unmarshal YAML file into test struct: %v", err)
+					return nil, fmt.Errorf("could not unmarshal YAML file into test struct: %v", err)
 				}
 				tests = append(tests, decoded)
 			case stateTestsFolderName:
 				decoded := &backend.StateTest{}
 				if err := yaml.Unmarshal(data, decoded); err != nil {
-					return nil, nil, fmt.Errorf("could not unmarshal YAML file into test struct: %v", err)
+					return nil, fmt.Errorf("could not unmarshal YAML file into test struct: %v", err)
 				}
 				tests = append(tests, decoded)
 
-			case configFilesFolderName:
-				decoded := &params.BeaconChainConfig{}
-				if err := yaml.Unmarshal(data, decoded); err != nil {
-					return nil, nil, fmt.Errorf("could not unmarshal YAML file into config struct: %v", err)
-				}
-				fileName := file.Name()
-				var extension = filepath.Ext(fileName)
-				var name = fileName[0 : len(fileName)-len(extension)]
-				configs[name] = decoded
 			}
 		}
 	}
-	return tests, configs, nil
+	return tests, nil
 }
 
-func runTests(tests []interface{}, configs map[string]interface{}, sb *backend.SimulatedBackend) error {
+func runTests(tests []interface{}, sb *backend.SimulatedBackend) error {
 	for _, tt := range tests {
 		switch typedTest := tt.(type) {
 		case *backend.ForkChoiceTest:
@@ -94,26 +72,6 @@ func runTests(tests []interface{}, configs map[string]interface{}, sb *backend.S
 				}
 			}
 			log.Info("Test PASSED")
-		case *backend.ShuffleTest:
-			log.Infof("Title: %v", typedTest.Title)
-			log.Infof("Summary: %v", typedTest.Summary)
-			log.Infof("Fork: %v", typedTest.Forks)
-			log.Infof("Config: %v", typedTest.Config)
-			config, ok := configs[string(typedTest.Config)]
-			if !ok {
-				return errors.New("no config file found for test")
-			}
-			conf, ok := config.(*params.BeaconChainConfig)
-			if !ok {
-				return fmt.Errorf("config file is not of type *params.BeaconChainConfig found type: %v", reflect.TypeOf(config))
-			}
-			params.OverrideBeaconConfig(conf)
-			for _, testCase := range typedTest.TestCases {
-				if err := sb.RunShuffleTest(testCase); err != nil {
-					return fmt.Errorf("shuffle test failed: %v", err)
-				}
-			}
-			log.Info("Test PASSED")
 		case *backend.StateTest:
 			log.Infof("Title: %v", typedTest.Title)
 			log.Infof("Summary: %v", typedTest.Summary)
@@ -143,7 +101,7 @@ func main() {
 	customFormatter.FullTimestamp = true
 	log.SetFormatter(customFormatter)
 
-	tests, configs, err := readTestsFromYaml(*yamlDir)
+	tests, err := readTestsFromYaml(*yamlDir)
 	if err != nil {
 		log.Fatalf("Fail to load tests from yaml: %v", err)
 	}
@@ -156,7 +114,7 @@ func main() {
 	log.Info("----Running Tests----")
 	startTime := time.Now()
 
-	err = runTests(tests, configs, sb)
+	err = runTests(tests, sb)
 	if err != nil {
 		log.Fatalf("Test failed %v", err)
 	}