BLS spec tests (#2826)

* bls spec tests

* add more bls tests

* use ioutil instead of bazel runfiles

* dont read bytes

* skip tests that overflow uint64

* manually fix input data

* add tests

* lint and gaz

* add all new changes

* some refactoring, cleanup, remove new API methods that only exist for tests

* gaz

* Remove yamls, skip test

WORKSPACE

@@ -129,6 +129,21 @@ http_archive(
     url = "https://github.com/kubernetes/repo-infra/archive/df02ded38f9506e5bbcbf21702034b4fef815f2f.tar.gz",
 )
 
+http_archive(
+    name = "eth2_spec_tests",
+    build_file_content = """
+filegroup(
+    name = "test_data",
+    srcs = glob([
+        "**/*.yaml",
+    ]),
+    visibility = ["//visibility:public"],
+)
+    """,
+    sha256 = "56847989737e816ab7d23f3bb2422347dfa81271bae81a94de512c01461fab25",
+    url = "https://github.com/prysmaticlabs/eth2.0-spec-tests/releases/download/v0.7.1/base64_encoded_archive.tar.gz",
+)
+
 http_archive(
     name = "com_github_bazelbuild_buildtools",
     strip_prefix = "buildtools-bf564b4925ab5876a3f64d8b90fab7f769013d42",

beacon-chain/main.go

@@ -16,7 +16,7 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/version"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
-	"github.com/x-cray/logrus-prefixed-formatter"
+	prefixed "github.com/x-cray/logrus-prefixed-formatter"
 	_ "go.uber.org/automaxprocs"
 )
 

shared/bls/spectest/BUILD.bazel

@@ -0,0 +1,39 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    testonly = True,
+    srcs = [
+        "aggregate_pubkeys_test.yaml.go",
+        "aggregate_sigs_test.yaml.go",
+        "g2_compressed_test.yaml.go",
+        "g2_uncompressed_test.yaml.go",
+        "priv_to_pub_test.yaml.go",
+        "sign_message_test.yaml.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/shared/bls/spectest",
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "aggregate_pubkeys_test.go",
+        "aggregate_sigs_test.go",
+        "g2_compressed_test.go",
+        "g2_uncompressed_test.go",
+        "helper_test.go",
+        "priv_to_pub_test.go",
+        "sign_message_test.go",
+    ],
+    data = ["@eth2_spec_tests//:test_data"],
+    embed = [":go_default_library"],
+    tags = ["spectest"],
+    deps = [
+        "//shared/bls:go_default_library",
+        "//shared/bytesutil:go_default_library",
+        "@com_github_ghodss_yaml//:go_default_library",
+        "@com_github_phoreproject_bls//:go_default_library",
+        "@io_bazel_rules_go//go/tools/bazel:go_default_library",
+    ],
+)

shared/bls/spectest/aggregate_pubkeys_test.go

@@ -0,0 +1,43 @@
+package spectest
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/ghodss/yaml"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+)
+
+func TestAggregatePubkeysYaml(t *testing.T) {
+	file, err := loadBlsYaml("aggregate_pubkeys/aggregate_pubkeys.yaml")
+	if err != nil {
+		t.Fatalf("Failed to read file: %v", err)
+	}
+
+	test := &AggregatePubkeysTest{}
+	if err := yaml.Unmarshal(file, test); err != nil {
+		t.Fatalf("Failed to unmarshal: %v", err)
+	}
+
+	for i, tt := range test.TestCases {
+		t.Run(fmt.Sprintf("Test %d", i), func(t *testing.T) {
+			pk, err := bls.PublicKeyFromBytes(tt.Input[0])
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, pk2 := range tt.Input[1:] {
+				p, err := bls.PublicKeyFromBytes(pk2)
+				if err != nil {
+					t.Fatal(err)
+				}
+				pk.Aggregate(p)
+			}
+
+			if !bytes.Equal(tt.Output, pk.Marshal()) {
+				t.Fatal("Output does not equal marshalled aggregated public " +
+					"key bytes")
+			}
+		})
+	}
+}

shared/bls/spectest/aggregate_sigs_test.go

@@ -0,0 +1,40 @@
+package spectest
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/shared/bls"
+
+	"github.com/ghodss/yaml"
+)
+
+func TestAggregateSignaturesYaml(t *testing.T) {
+	file, err := loadBlsYaml("aggregate_sigs/aggregate_sigs.yaml")
+	if err != nil {
+		t.Fatalf("Failed to read file: %v", err)
+	}
+
+	test := &AggregateSigsTest{}
+	if err := yaml.Unmarshal(file, test); err != nil {
+		t.Fatalf("Failed to unmarshal: %v", err)
+	}
+
+	for i, tt := range test.TestCases {
+		t.Run(fmt.Sprintf("Test %d", i), func(t *testing.T) {
+			var sigs []*bls.Signature
+			for _, s := range tt.Input {
+				sig, err := bls.SignatureFromBytes(s)
+				if err != nil {
+					t.Fatalf("Unable to unmarshal signature from bytes: %v", err)
+				}
+				sigs = append(sigs, sig)
+			}
+			sig := bls.AggregateSignatures(sigs)
+			if !bytes.Equal(tt.Output, sig.Marshal()) {
+				t.Fatal("Output does not equal marshalled aggregated sig bytes")
+			}
+		})
+	}
+}

shared/bls/spectest/g2_compressed_test.go

@@ -0,0 +1,47 @@
+package spectest
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/ghodss/yaml"
+	"github.com/phoreproject/bls"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+)
+
+// Note: This actually tests the underlying library as we don't have a need for
+// HashG2Compressed in our local BLS API.
+func TestG2CompressedHash(t *testing.T) {
+	file, err := loadBlsYaml("msg_hash_g2_compressed/g2_compressed.yaml")
+	if err != nil {
+		t.Fatalf("Failed to read file: %v", err)
+	}
+
+	test := &G2CompressedTest{}
+	if err := yaml.Unmarshal(file, test); err != nil {
+		t.Fatalf("Failed to unmarshal: %v", err)
+	}
+
+	for i, tt := range test.TestCases {
+		t.Run(fmt.Sprintf("Test %d", i), func(t *testing.T) {
+
+			projective := bls.HashG2WithDomain(
+				bytesutil.ToBytes32(tt.Input.Message),
+				tt.Input.Domain,
+			)
+			hash := bls.CompressG2(projective.ToAffine())
+
+			var buf []byte
+			for _, slice := range tt.Output {
+				buf = append(buf, slice...)
+			}
+			if !bytes.Equal(buf, hash[:]) {
+				t.Logf("Domain=%d", tt.Input.Domain)
+				t.Fatalf("Hash does not match the expected output. "+
+					"Expected %#x but received %#x", buf, hash)
+			}
+			t.Logf("Success. Domain=%d", tt.Input.Domain)
+		})
+	}
+}

shared/bls/spectest/g2_uncompressed_test.go

@@ -0,0 +1,49 @@
+package spectest
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/ghodss/yaml"
+	"github.com/phoreproject/bls"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+)
+
+// Note: This actually tests the underlying library as we don't have a need for
+// HashG2Uncompressed in our local BLS API.
+func TestG2UncompressedHash(t *testing.T) {
+	t.Skip("The python uncompressed method does not match the go uncompressed method and this isn't very important")
+	file, err := loadBlsYaml("msg_hash_g2_uncompressed/g2_uncompressed.yaml")
+	if err != nil {
+		t.Fatalf("Failed to read file: %v", err)
+	}
+
+	test := &G2UncompressedTest{}
+	if err := yaml.Unmarshal(file, test); err != nil {
+		t.Fatalf("Failed to unmarshal: %v", err)
+	}
+
+	for i, tt := range test.TestCases {
+		t.Run(fmt.Sprintf("Test %d", i), func(t *testing.T) {
+			projective := bls.HashG2WithDomain(
+				bytesutil.ToBytes32(tt.Input.Message),
+				tt.Input.Domain,
+			)
+			hash := projective.ToAffine().SerializeBytes()
+
+			var buf []byte
+			for _, slice := range tt.Output {
+				for _, innerSlice := range slice {
+					buf = append(buf, innerSlice...)
+				}
+			}
+			if !bytes.Equal(buf, hash[:]) {
+				t.Logf("Domain=%d", tt.Input.Domain)
+				t.Fatalf("Hash does not match the expected output. "+
+					"Expected %#x but received %#x", buf, hash)
+			}
+			t.Logf("Success. Domain=%d", tt.Input.Domain)
+		})
+	}
+}

shared/bls/spectest/helper_test.go

@@ -0,0 +1,21 @@
+package spectest
+
+import (
+	"io/ioutil"
+
+	"github.com/bazelbuild/rules_go/go/tools/bazel"
+)
+
+const prefix = "/eth2_spec_tests/tests/bls/"
+
+// Load BLS yaml from spec test bls directory. The file parameter should be in
+// the format of the path starting at the bls directory.
+// Example: aggregate_pubkeys/aggregate_pubkeys.yaml where the full path would
+// be tests/bls/aggregate_pubkeys/aggregate_pubkeys.yaml.
+func loadBlsYaml(file string) ([]byte, error) {
+	filepath, err := bazel.Runfile(prefix + file)
+	if err != nil {
+		return []byte{}, err
+	}
+	return ioutil.ReadFile(filepath)
+}

shared/bls/spectest/priv_to_pub_test.go

@@ -0,0 +1,34 @@
+package spectest
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/ghodss/yaml"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+)
+
+func TestPrivToPubYaml(t *testing.T) {
+	file, err := loadBlsYaml("priv_to_pub/priv_to_pub.yaml")
+	if err != nil {
+		t.Fatalf("Failed to read file: %v", err)
+	}
+
+	test := &PrivToPubTest{}
+	if err := yaml.Unmarshal(file, test); err != nil {
+		t.Fatalf("Failed to unmarshal: %v", err)
+	}
+
+	for i, tt := range test.TestCases {
+		t.Run(fmt.Sprintf("Test %d", i), func(t *testing.T) {
+			sk, err := bls.SecretKeyFromBytes(tt.Input)
+			if err != nil {
+				t.Fatalf("Cannot unmarshal input to secret key: %v", err)
+			}
+			if !bytes.Equal(tt.Output, sk.PublicKey().Marshal()) {
+				t.Fatal("Output does not marshalled public key bytes")
+			}
+		})
+	}
+}