Select entrypoint command based on runtime platform

This fixes a long-standing bug affecting heterogenous clusters, where
the controller's platform would be used to lookup the image's
entrypoint, instead of the platform of the node where the workload would
eventually run.

With this change, the controller looks up _all_ the image's entrypoints
and passes them to the entrypoint binary on the node, where it uses its
current runtime platform to lookup the correct entrypoint to execute.

This has the added benefit that we can now pass the entire image@digest
of the multi-platform image down to the Pod, instead of the
(controller's) platform-specific image. This has benefits for scenarios
where Pods may be blocked from running unsigned/untrusted images, since
it might be the multi-platform image index that's signed/trusted, and
not any particular platform-specific constituent image.

cmd/entrypoint/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"log"
 	"os"
@@ -25,6 +26,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containerd/containerd/platforms"
 	"github.com/tektoncd/pipeline/cmd/entrypoint/subcommands"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline"
 	"github.com/tektoncd/pipeline/pkg/credentials"
@@ -99,13 +101,36 @@ func main() {
 		}
 	}
 
+	var cmd []string
+	if *ep != "" {
+		cmd = []string{*ep}
+	} else {
+		env := os.Getenv("TEKTON_PLATFORM_COMMANDS")
+		var cmds map[string][]string
+		if err := json.Unmarshal([]byte(env), &cmds); err != nil {
+			log.Fatal(err)
+		}
+		log.Println("COMMANDS: %+v", cmds)
+		plat := platforms.DefaultString()
+
+		var found bool
+		cmd, found = cmds[plat]
+		if !found {
+			// The image might not be multi-platform, in which case
+			// there's only one value for all runtime platforms.
+			cmd, found = cmds["*"]
+		}
+		if !found {
+			log.Fatalf("could not find command for platform %q", plat)
+		}
+	}
+
 	e := entrypoint.Entrypointer{
-		Entrypoint:          *ep,
+		Command:             append(cmd, flag.Args()...),
 		WaitFiles:           strings.Split(*waitFiles, ","),
 		WaitFileContent:     *waitFileContent,
 		PostFile:            *postFile,
 		TerminationPath:     *terminationPath,
-		Args:                flag.Args(),
 		Waiter:              &realWaiter{waitPollingInterval: defaultWaitPollingInterval, breakpointOnFailure: *breakpointOnFailure},
 		Runner:              &realRunner{},
 		PostWriter:          &realPostWriter{},

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/cloudevents/sdk-go/v2 v2.5.0
+	github.com/containerd/containerd v1.5.2
 	github.com/docker/cli v20.10.8+incompatible // indirect
 	github.com/docker/docker v20.10.8+incompatible // indirect
 	github.com/emicklei/go-restful v2.15.0+incompatible // indirect
@@ -15,9 +16,10 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/googleapis/gnostic v0.5.3 // indirect
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/hashicorp/golang-lru v0.5.4
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/jenkins-x/go-scm v1.10.10
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/opencontainers/image-spec v1.0.3-0.20211202222133-eacdcc10569b
 	github.com/pkg/errors v0.9.1
 	github.com/tektoncd/plumbing v0.0.0-20211012143332-c7cc43d9bc0c
 	go.opencensus.io v0.23.0

pkg/entrypoint/entrypointer.go

@@ -45,10 +45,9 @@ const (
 // Entrypointer holds fields for running commands with redirected
 // entrypoints.
 type Entrypointer struct {
-	// Entrypoint is the original specified entrypoint, if any.
-	Entrypoint string
-	// Args are the original specified args, if any.
-	Args []string
+	// Command is the original specified command and args.
+	Command []string
+
 	// WaitFiles is the set of files to wait for. If empty, execution
 	// begins immediately.
 	WaitFiles []string
@@ -141,10 +140,6 @@ func (e Entrypointer) Go() error {
 		}
 	}
 
-	if e.Entrypoint != "" {
-		e.Args = append([]string{e.Entrypoint}, e.Args...)
-	}
-
 	output = append(output, v1beta1.PipelineResourceResult{
 		Key:        "StartedAt",
 		Value:      time.Now().Format(timeFormat),
@@ -163,7 +158,7 @@ func (e Entrypointer) Go() error {
 			ctx, cancel = context.WithTimeout(ctx, *e.Timeout)
 			defer cancel()
 		}
-		err = e.Runner.Run(ctx, e.Args...)
+		err = e.Runner.Run(ctx, e.Command...)
 		if err == context.DeadlineExceeded {
 			output = append(output, v1beta1.PipelineResourceResult{
 				Key:        "Reason",

pkg/pod/entrypoint.go

@@ -147,15 +147,6 @@ func orderContainers(commonExtraEntrypointArgs []string, steps []corev1.Containe
 			argsForEntrypoint = append(argsForEntrypoint, resultArgument(steps, taskSpec.Results)...)
 		}
 
-		cmd, args := s.Command, s.Args
-		if len(cmd) == 0 {
-			return nil, fmt.Errorf("Step %d did not specify command", i)
-		}
-		if len(cmd) > 1 {
-			args = append(cmd[1:], args...)
-			cmd = []string{cmd[0]}
-		}
-
 		if breakpointConfig != nil && len(breakpointConfig.Breakpoint) > 0 {
 			breakpoints := breakpointConfig.Breakpoint
 			for _, b := range breakpoints {
@@ -166,7 +157,13 @@ func orderContainers(commonExtraEntrypointArgs []string, steps []corev1.Containe
 			}
 		}
 
-		argsForEntrypoint = append(argsForEntrypoint, "-entrypoint", cmd[0], "--")
+		cmd, args := s.Command, s.Args
+		if len(cmd) > 0 {
+			argsForEntrypoint = append(argsForEntrypoint, "-entrypoint", cmd[0], "--")
+		}
+		if len(cmd) > 1 {
+			args = append(cmd[1:], args...)
+		}
 		argsForEntrypoint = append(argsForEntrypoint, args...)
 
 		steps[i].Command = []string{entrypointBinary}

pkg/pod/entrypoint_lookup.go

@@ -18,7 +18,7 @@ package pod
 
 import (
 	"context"
-	"fmt"
+	"encoding/json"
 
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -31,10 +31,18 @@ type EntrypointCache interface {
 	// Get the Image data for the given image reference. If the value is
 	// not found in the cache, it will be fetched from the image registry,
 	// possibly using K8s service account imagePullSecrets.
-	Get(ctx context.Context, ref name.Reference, namespace, serviceAccountName string) (v1.Image, error)
-	// Set updates the cache with a new digest->Image mapping. This will avoid a
-	// remote registry lookup next time Get is called.
-	Set(digest name.Digest, img v1.Image)
+	//
+	// It also returns the digest associated with the given reference. If
+	// the reference referred to an index, the returned digest will be the
+	// index's digest, not any platform-specific image contained by the
+	// index.
+	Get(ctx context.Context, ref name.Reference, namespace, serviceAccountName string) (*imageData, error)
+}
+
+// imageData contains information looked up about an image or multi-platform image index.
+type imageData struct {
+	digest   v1.Hash
+	commands map[string][]string // map of platform -> []command
 }
 
 // resolveEntrypoints looks up container image ENTRYPOINTs for all steps that
@@ -43,70 +51,49 @@ type EntrypointCache interface {
 // Images that are not specified by digest will be specified by digest after
 // lookup in the resulting list of containers.
 func resolveEntrypoints(ctx context.Context, cache EntrypointCache, namespace, serviceAccountName string, steps []corev1.Container) ([]corev1.Container, error) {
-	// Keep a local cache of name->image lookups, just for the scope of
+	// Keep a local cache of name->imageData lookups, just for the scope of
 	// resolving this set of steps. If the image is pushed to before the
-	// next run, we need to resolve its digest and entrypoint again, but we
+	// next run, we need to resolve its digest and commands again, but we
 	// can skip lookups while resolving the same TaskRun.
-	localCache := map[name.Reference]v1.Image{}
+	localCache := map[name.Reference]imageData{}
 	for i, s := range steps {
-		if len(s.Command) != 0 {
-			// Nothing to resolve.
+
+		// If the command is already specified, there's nothing to resolve.
+		if len(s.Command) > 0 {
 			continue
 		}
 
-		origRef, err := name.ParseReference(s.Image, name.WeakValidation)
+		ref, err := name.ParseReference(s.Image, name.WeakValidation)
 		if err != nil {
 			return nil, err
 		}
-		var img v1.Image
-		if cimg, found := localCache[origRef]; found {
-			img = cimg
+		var id imageData
+		if cid, found := localCache[ref]; found {
+			id = cid
 		} else {
-			// Look it up in the cache. If it's not found in the
-			// cache, it will be resolved from the registry.
-			img, err = cache.Get(ctx, origRef, namespace, serviceAccountName)
+			// Look it up for real.
+			lid, err := cache.Get(ctx, ref, namespace, serviceAccountName)
 			if err != nil {
 				return nil, err
 			}
-			// Cache it locally in case another step specifies the same image.
-			localCache[origRef] = img
+			id = *lid
+
+			// Cache it locally in case another step in this task specifies the same image.
+			localCache[ref] = *lid
 		}
 
-		ep, digest, err := imageData(origRef, img)
+		// Resolve the original reference to a reference by digest.
+		steps[i].Image = ref.Context().Digest(id.digest.String()).String()
+
+		// Encode the map of platform->command to JSON and pass it via env var.
+		b, err := json.Marshal(id.commands)
 		if err != nil {
 			return nil, err
 		}
-
-		cache.Set(digest, img) // Cache the lookup for next time this image is looked up by digest.
-
-		steps[i].Image = digest.String() // Specify image by digest, since we know it now.
-		steps[i].Command = ep            // Specify the command explicitly.
+		steps[i].Env = append(steps[i].Env, corev1.EnvVar{
+			Name:  "TEKTON_PLATFORM_COMMANDS",
+			Value: string(b),
+		})
 	}
 	return steps, nil
 }
-
-// imageData pulls the entrypoint from the image, and returns the given
-// original reference, with image digest resolved.
-func imageData(ref name.Reference, img v1.Image) ([]string, name.Digest, error) {
-	digest, err := img.Digest()
-	if err != nil {
-		return nil, name.Digest{}, fmt.Errorf("error getting image digest: %v", err)
-	}
-	cfg, err := img.ConfigFile()
-	if err != nil {
-		return nil, name.Digest{}, fmt.Errorf("error getting image config: %v", err)
-	}
-
-	// Entrypoint can be specified in either .Config.Entrypoint or
-	// .Config.Cmd.
-	ep := cfg.Config.Entrypoint
-	if len(ep) == 0 {
-		ep = cfg.Config.Cmd
-	}
-
-	d, err := name.NewDigest(ref.Context().String()+"@"+digest.String(), name.WeakValidation)
-	if err != nil {
-		return nil, name.Digest{}, fmt.Errorf("error constructing resulting digest: %v", err)
-	}
-	return ep, d, nil
-}