diff --git a/deps.bzl b/deps.bzl index 8e9407a717b..9f2cfa29b2c 100644 --- a/deps.bzl +++ b/deps.bzl @@ -3933,8 +3933,8 @@ def install_buildbuddy_dependencies(workspace_name = "buildbuddy"): go_repository( name = "com_github_minio_sha256_simd", importpath = "github.com/minio/sha256-simd", - sum = "h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=", - version = "v1.0.0", + sum = "h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=", + version = "v1.0.1", ) go_repository( @@ -6860,8 +6860,8 @@ def install_buildbuddy_dependencies(workspace_name = "buildbuddy"): go_repository( name = "org_golang_x_sys", importpath = "golang.org/x/sys", - sum = "h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=", - version = "v0.10.0", + sum = "h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=", + version = "v0.11.0", ) go_repository( name = "org_golang_x_term", diff --git a/go.mod b/go.mod index 9864d20e0ed..7c6beb9c8fc 100644 --- a/go.mod +++ b/go.mod @@ -119,7 +119,7 @@ require ( golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/oauth2 v0.9.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.10.0 + golang.org/x/sys v0.11.0 golang.org/x/text v0.11.0 golang.org/x/time v0.3.0 google.golang.org/api v0.129.0 @@ -277,6 +277,7 @@ require ( github.com/mdlayher/socket v0.4.1 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/miekg/dns v1.1.55 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/go.sum b/go.sum index 160bbbc756b..98c8b284f3c 100644 --- a/go.sum +++ b/go.sum @@ -1938,6 +1938,8 @@ github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= @@ -2914,6 +2916,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/server/remote_cache/digest/BUILD b/server/remote_cache/digest/BUILD index 3039984e8cd..eefc29e82d3 100644 --- a/server/remote_cache/digest/BUILD +++ b/server/remote_cache/digest/BUILD @@ -1,5 +1,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +# gazelle:ignore + go_library( name = "digest", srcs = ["digest.go"], @@ -23,13 +25,44 @@ go_library( go_test( name = "digest_test", size = "small", - srcs = ["digest_test.go"], + srcs = [ + "digest_test.go", + ] + select({ + "@platforms//cpu:x86_64": ["digest_amd64_test.go"], + "//conditions:default": [], + }), + embed = [":digest"], + deps = [ + "//proto:remote_execution_go_proto", + "//proto:resource_go_proto", + "//server/util/compression", + "//server/util/status", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//status", + ] + select({ + "@io_bazel_rules_go//go/platform:amd64": [ + "@com_github_minio_sha256_simd//:sha256-simd", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "simd_bench_test", + srcs = ["digest_amd64_test.go"], + args = [ + "-test.bench=BenchmarkSIMDDigestCompute", + ], embed = [":digest"], + exec_compatible_with = ["@platforms//cpu:x86_64"], + tags = ["manual"], deps = [ "//proto:remote_execution_go_proto", "//proto:resource_go_proto", "//server/util/compression", "//server/util/status", + "@com_github_minio_sha256_simd//:sha256-simd", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@org_golang_google_grpc//status", diff --git a/server/remote_cache/digest/digest_amd64_test.go b/server/remote_cache/digest/digest_amd64_test.go new file mode 100644 index 00000000000..a0ac95cbf01 --- /dev/null +++ b/server/remote_cache/digest/digest_amd64_test.go @@ -0,0 +1,78 @@ +//go:build amd64 + +package digest_test + +import ( + "bytes" + "crypto/sha256" + "fmt" + "hash" + "io" + "math/rand" + "testing" + + sha256simd "github.com/minio/sha256-simd" + "github.com/stretchr/testify/require" +) + +var server = sha256simd.NewAvx512Server() + +func hasherWithServer() hash.Hash { + return sha256simd.NewAvx512(server) +} + +func BenchmarkSIMDDigestCompute(b *testing.B) { + for _, size := range []int{ + // 1 MB + 1_000_000, + // 10 MB + 10_000_000, + // 100 MB + 100_000_000, + // 1 GB + 1_000_000_000, + // 10 GB + 10_000_000_000, + } { + buf := make([]byte, size) + _, err := rand.Read(buf) + require.NoError(b, err) + + for _, tc := range []struct { + newHasher func() hash.Hash + name string + }{ + {sha256.New, "without_SIMD"}, + {sha256simd.New, "with_SIMD_no_server"}, + {hasherWithServer, "with_SIMD_with_server"}, + } { + b.Run(fmt.Sprintf("%s/%d", tc.name, size), func(b *testing.B) { + for i := 0; i < b.N; i++ { + tc.newHasher().Sum(buf) + } + }) + } + } +} + +func TestHashStableOutput(t *testing.T) { + for _, size := range []int{1, 10, 100, 1000, 10_000, 100_000} { + t.Run(fmt.Sprintf("size_%d", size), func(t *testing.T) { + buf := make([]byte, size) + _, err := rand.Read(buf) + require.NoError(t, err) + + lastSum := []byte{} + for i, h := range []hash.Hash{sha256.New(), sha256simd.New(), hasherWithServer()} { + _, err := io.Copy(h, bytes.NewReader(buf)) + require.NoError(t, err) + + curSum := h.Sum([]byte{}) + if i > 0 { + require.Equal(t, lastSum, curSum) + } + lastSum = curSum + } + }) + } +}