remote_cache/digest: add benchmark for sha256-simd

deps.bzl

@@ -3933,8 +3933,8 @@ def install_buildbuddy_dependencies(workspace_name = "buildbuddy"):
     go_repository(
         name = "com_github_minio_sha256_simd",
         importpath = "github.com/minio/sha256-simd",
-        sum = "h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=",
-        version = "v1.0.0",
+        sum = "h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=",
+        version = "v1.0.1",
     )
 
     go_repository(
@@ -6860,8 +6860,8 @@ def install_buildbuddy_dependencies(workspace_name = "buildbuddy"):
     go_repository(
         name = "org_golang_x_sys",
         importpath = "golang.org/x/sys",
-        sum = "h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=",
-        version = "v0.10.0",
+        sum = "h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=",
+        version = "v0.11.0",
     )
     go_repository(
         name = "org_golang_x_term",

go.mod

@@ -119,7 +119,7 @@ require (
 	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
 	golang.org/x/oauth2 v0.9.0
 	golang.org/x/sync v0.3.0
-	golang.org/x/sys v0.10.0
+	golang.org/x/sys v0.11.0
 	golang.org/x/text v0.11.0
 	golang.org/x/time v0.3.0
 	google.golang.org/api v0.129.0
@@ -277,6 +277,7 @@ require (
 	github.com/mdlayher/socket v0.4.1 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/miekg/dns v1.1.55 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect

go.sum

@@ -1938,6 +1938,8 @@ github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WT
 github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
 github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
+github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
+github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
@@ -2914,6 +2916,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

server/remote_cache/digest/BUILD

@@ -1,5 +1,7 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
+# gazelle:ignore
+
 go_library(
     name = "digest",
     srcs = ["digest.go"],
@@ -23,13 +25,44 @@ go_library(
 go_test(
     name = "digest_test",
     size = "small",
-    srcs = ["digest_test.go"],
+    srcs = [
+        "digest_test.go",
+    ] + select({
+        "@platforms//cpu:x86_64": ["digest_amd64_test.go"],
+        "//conditions:default": [],
+    }),
+    embed = [":digest"],
+    deps = [
+        "//proto:remote_execution_go_proto",
+        "//proto:resource_go_proto",
+        "//server/util/compression",
+        "//server/util/status",
+        "@com_github_stretchr_testify//assert",
+        "@com_github_stretchr_testify//require",
+        "@org_golang_google_grpc//status",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:amd64": [
+            "@com_github_minio_sha256_simd//:sha256-simd",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_test(
+    name = "simd_bench_test",
+    srcs = ["digest_amd64_test.go"],
+    args = [
+        "-test.bench=BenchmarkSIMDDigestCompute",
+    ],
     embed = [":digest"],
+    exec_compatible_with = ["@platforms//cpu:x86_64"],
+    tags = ["manual"],
     deps = [
         "//proto:remote_execution_go_proto",
         "//proto:resource_go_proto",
         "//server/util/compression",
         "//server/util/status",
+        "@com_github_minio_sha256_simd//:sha256-simd",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",
         "@org_golang_google_grpc//status",

server/remote_cache/digest/digest_amd64_test.go

@@ -0,0 +1,78 @@
+//go:build amd64
+
+package digest_test
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"io"
+	"math/rand"
+	"testing"
+
+	sha256simd "github.com/minio/sha256-simd"
+	"github.com/stretchr/testify/require"
+)
+
+var server = sha256simd.NewAvx512Server()
+
+func hasherWithServer() hash.Hash {
+	return sha256simd.NewAvx512(server)
+}
+
+func BenchmarkSIMDDigestCompute(b *testing.B) {
+	for _, size := range []int{
+		// 1 MB
+		1_000_000,
+		// 10 MB
+		10_000_000,
+		// 100 MB
+		100_000_000,
+		// 1 GB
+		1_000_000_000,
+		// 10 GB
+		10_000_000_000,
+	} {
+		buf := make([]byte, size)
+		_, err := rand.Read(buf)
+		require.NoError(b, err)
+
+		for _, tc := range []struct {
+			newHasher func() hash.Hash
+			name      string
+		}{
+			{sha256.New, "without_SIMD"},
+			{sha256simd.New, "with_SIMD_no_server"},
+			{hasherWithServer, "with_SIMD_with_server"},
+		} {
+			b.Run(fmt.Sprintf("%s/%d", tc.name, size), func(b *testing.B) {
+				for i := 0; i < b.N; i++ {
+					tc.newHasher().Sum(buf)
+				}
+			})
+		}
+	}
+}
+
+func TestHashStableOutput(t *testing.T) {
+	for _, size := range []int{1, 10, 100, 1000, 10_000, 100_000} {
+		t.Run(fmt.Sprintf("size_%d", size), func(t *testing.T) {
+			buf := make([]byte, size)
+			_, err := rand.Read(buf)
+			require.NoError(t, err)
+
+			lastSum := []byte{}
+			for i, h := range []hash.Hash{sha256.New(), sha256simd.New(), hasherWithServer()} {
+				_, err := io.Copy(h, bytes.NewReader(buf))
+				require.NoError(t, err)
+
+				curSum := h.Sum([]byte{})
+				if i > 0 {
+					require.Equal(t, lastSum, curSum)
+				}
+				lastSum = curSum
+			}
+		})
+	}
+}