Shuffle tests revisited

beacon-chain/chaintest/backend/simulated_backend.go

@@ -173,30 +173,6 @@ func (sb *SimulatedBackend) RunForkChoiceTest(testCase *ForkChoiceTestCase) erro
 	return nil
 }
 
-// RunShuffleTest uses validator set specified from a YAML file, runs the validator shuffle
-// algorithm, then compare the output with the expected output from the YAML file.
-func (sb *SimulatedBackend) RunShuffleTest(testCase *ShuffleTestCase) error {
-	defer db.TeardownDB(sb.beaconDB)
-	seed := common.HexToHash(testCase.Seed)
-	testIndices := make([]uint64, testCase.Count, testCase.Count)
-	for i := uint64(0); i < testCase.Count; i++ {
-		testIndices[i] = i
-	}
-	shuffledList := make([]uint64, testCase.Count)
-	for i := uint64(0); i < testCase.Count; i++ {
-		si, err := utils.ShuffledIndex(i, testCase.Count, seed)
-		if err != nil {
-			log.Error(err)
-			return err
-		}
-		shuffledList[i] = si
-	}
-	if !reflect.DeepEqual(shuffledList, testCase.Shuffled) {
-		return fmt.Errorf("shuffle result error: expected %v, actual %v", testCase.Shuffled, shuffledList)
-	}
-	return nil
-}
-
 // RunStateTransitionTest advances a beacon chain state transition an N amount of
 // slots from a genesis state, with a block being processed at every iteration
 // of the state transition function.

beacon-chain/utils/spectest/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["shuffle_test_format.go"],
+    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/utils/spectest",
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["shuffle_yaml_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//beacon-chain/utils:go_default_library",
+        "//shared/params/spectest:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_go_yaml_yaml//:go_default_library",
+    ],
+)

beacon-chain/chaintest/backend/shuffle_test_format.go → beacon-chain/utils/spectest/shuffle_test_format.go

@@ -1,4 +1,4 @@
-package backend
+package spectest
 
 // ShuffleTest --
 type ShuffleTest struct {

beacon-chain/utils/spectest/shuffle_yaml_test.go

@@ -0,0 +1,84 @@
+package spectest
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/go-yaml/yaml"
+	"github.com/prysmaticlabs/prysm/beacon-chain/utils"
+	"github.com/prysmaticlabs/prysm/shared/params/spectest"
+)
+
+func TestShuffleYaml(t *testing.T) {
+	yamlDir := "./"
+	ext := ".yaml"
+	files, err := ioutil.ReadDir(yamlDir)
+	if err != nil {
+		t.Fatalf("could not read YAML tests directory: %v", err)
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		extension := string(file.Name()[len(file.Name())-5:])
+		if err != nil || extension != ext {
+			continue
+		}
+		filePath := path.Join(yamlDir, file.Name())
+		// #nosec G304
+		data, err := ioutil.ReadFile(filePath)
+		if err != nil {
+			t.Fatalf("could not read YAML tests directory: %v", err)
+		}
+		shuffleTest := &ShuffleTest{}
+		if err := yaml.Unmarshal(data, shuffleTest); err != nil {
+			t.Fatalf("could not unmarshal YAML file into test struct: %v", err)
+		}
+		if shuffleTest.Config == "minimal" {
+			if err := spectest.SetConfig("minimal"); err != nil {
+				t.Fatal(err)
+			}
+		} else {
+			if err := spectest.SetConfig("mainnet"); err != nil {
+				t.Fatal(err)
+			}
+		}
+		t.Logf("Title: %v", shuffleTest.Title)
+		t.Logf("Summary: %v", shuffleTest.Summary)
+		t.Logf("Fork: %v", shuffleTest.Forks)
+		t.Logf("Config: %v", shuffleTest.Config)
+		for _, testCase := range shuffleTest.TestCases {
+			if err := runShuffleTest(testCase); err != nil {
+				t.Fatalf("shuffle test failed: %v", err)
+			}
+		}
+
+	}
+}
+
+// RunShuffleTest uses validator set specified from a YAML file, runs the validator shuffle
+// algorithm, then compare the output with the expected output from the YAML file.
+func runShuffleTest(testCase *ShuffleTestCase) error {
+	seed := common.HexToHash(testCase.Seed)
+	testIndices := make([]uint64, testCase.Count, testCase.Count)
+	for i := uint64(0); i < testCase.Count; i++ {
+		testIndices[i] = i
+	}
+	shuffledList := make([]uint64, testCase.Count)
+	for i := uint64(0); i < testCase.Count; i++ {
+		si, err := utils.ShuffledIndex(i, testCase.Count, seed)
+		if err != nil {
+			return err
+		}
+		shuffledList[i] = si
+	}
+	if !reflect.DeepEqual(shuffledList, testCase.Shuffled) {
+		return fmt.Errorf("shuffle result error: expected %v, actual %v", testCase.Shuffled, shuffledList)
+	}
+	return nil
+}