diff --git a/session/sshforward/sshprovider/agentprovider.go b/session/sshforward/sshprovider/agentprovider.go
index 4f569799f7b9a..f8ed2811da4ac 100644
--- a/session/sshforward/sshprovider/agentprovider.go
+++ b/session/sshforward/sshprovider/agentprovider.go
@@ -6,6 +6,8 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"runtime"
+	"strings"
 	"time"
 
 	"github.com/moby/buildkit/session"
@@ -139,7 +141,7 @@ func toAgentSource(paths []string) (source, error) {
 			socket = p
 			continue
 		}
-		keys = true
+
 		f, err := os.Open(p)
 		if err != nil {
 			return source{}, errors.Wrapf(err, "failed to open %s", p)
@@ -151,11 +153,24 @@ func toAgentSource(paths []string) (source, error) {
 
 		k, err := ssh.ParseRawPrivateKey(dt)
 		if err != nil {
+			// On Windows, os.ModeSocket isn't appropriately set on the file mode.
+			// https://github.com/golang/go/issues/33357
+			// If parsing the file fails, check to see if it kind of looks like socket-shaped.
+			if runtime.GOOS == "windows" && strings.Contains(string(dt), "socket") {
+				if keys {
+					return source{}, errors.Errorf("invalid combination of keys and sockets")
+				}
+				socket = p
+				continue
+			}
+
 			return source{}, errors.Wrapf(err, "failed to parse %s", p) // TODO: prompt passphrase?
 		}
 		if err := a.Add(agent.AddedKey{PrivateKey: k}); err != nil {
 			return source{}, errors.Wrapf(err, "failed to add %s to agent", p)
 		}
+
+		keys = true
 	}
 
 	if socket != "" {
diff --git a/session/sshforward/sshprovider/agentprovider_test.go b/session/sshforward/sshprovider/agentprovider_test.go
new file mode 100644
index 0000000000000..733fe9ffecfc4
--- /dev/null
+++ b/session/sshforward/sshprovider/agentprovider_test.go
@@ -0,0 +1,21 @@
+package sshprovider_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/moby/buildkit/cmd/buildctl/build"
+	"github.com/moby/buildkit/session/sshforward/sshprovider"
+)
+
+func TestToAgentSource(t *testing.T) {
+	configs, err := build.ParseSSH([]string{"default"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = sshprovider.NewSSHAgentProvider(configs)
+	ok := err == nil || strings.Contains(err.Error(), "invalid empty ssh agent socket")
+	if !ok {
+		t.Fatal(err)
+	}
+}