From 343d60ec3678247f2ddb3283d2c18b2495e1b4e9 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Tue, 26 Jul 2022 06:51:09 +0000 Subject: [PATCH 01/13] Add a simple runner API to run commands --- internal/runner/runner.go | 143 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 internal/runner/runner.go diff --git a/internal/runner/runner.go b/internal/runner/runner.go new file mode 100644 index 0000000000..903803553c --- /dev/null +++ b/internal/runner/runner.go @@ -0,0 +1,143 @@ +package runner + +import ( + "errors" + "os" + "path/filepath" + "strings" + "syscall" +) + +// CommandRunner runs commands and returns the build steps that were run. +type CommandRunner struct { + // Env is global environment variables passed to all commands. + Env []string + + // Steps are the steps to execute. + Steps []*CommandStep +} + +// CommandStep is a command that was executed by the builder. +type CommandStep struct { + // Command is the command that was run. + Command []string `json:"command"` + + // Env are the environment variables passed to the command. + Env []string `json:"env"` + + // WorkingDir is the working directory the command was executed in. + WorkingDir string `json:"workingDir"` +} + +// Run excutes a series of commands and returns the steps that were executed +// successfully. Commands are run in sequence and are expected to return a zero +// exit status. +// +// Global environment variables are merged with steps environment variables in +// the returned steps. In the case of duplicates the last occurrence has precidence. +// Environment variables are *not* inherited from the current process. +// +// The returned CommandSteps should be included in the buildConfig provenance. +// These are *not* the same as the runner commands. Env vars are sanitized, pwd +// is changed to the absolute path, and only commands that executed +// successfully are returned. +func (r *CommandRunner) Run() (steps []*CommandStep, err error) { + var originalwd string + originalwd, err = os.Getwd() + if err != nil { + return // steps, err + } + defer func() { + // Change directories back to the original working directory but only + // return the error returned by Chdir if no other error occurred. + if chDirErr := os.Chdir(originalwd); err == nil { + // NOTE: err is returned by the function after the defer is called. + err = chDirErr + } + }() + + for _, step := range r.Steps { + if len(step.Command) == 0 { + err = errors.New("command is empty") + return // steps, err + } + + cmd := step.Command[0] + args := step.Command[1:] + // TODO: Add some kind of LD_PRELOAD protection? + env := dedupEnv(append(r.Env, step.Env...)) + + // Set the POSIX PWD env var. + var pwd string + pwd, err = filepath.Abs(step.WorkingDir) + if err != nil { + return // steps, err + } + env = append(env, "PWD="+pwd) + + err = os.Chdir(pwd) + if err != nil { + return // steps, err + } + + // NOTE: We use syscall.Exec directly because we are executing the + // commands in sequence and we want full control over the environment. + // The stdlib exec package deduplicates env vars and so it's hard to + // know exactly what the command was run with. + // NOTE: The executed command will inherit stdout/stderr. + err = syscall.Exec(cmd, args, env) + if err != nil { + return // steps, err + } + + steps = append(steps, &CommandStep{ + Command: append([]string{cmd}, args...), + Env: env, + WorkingDir: pwd, + }) + } + return // steps, err +} + +// dedupEnv returns a copy of env with any duplicates removed, in favor of +// later values. +// Items not of the normal environment "key=value" form are preserved unchanged. +// NOTE: adapted from the stdlib exec package. +func dedupEnv(env []string) []string { + // Construct the output in reverse order, to preserve the + // last occurrence of each key. + out := make([]string, 0, len(env)) + saw := make(map[string]bool, len(env)) + for n := len(env); n > 0; n-- { + kv := env[n-1] + + i := strings.Index(kv, "=") + if i == 0 { + // We observe in practice keys with a single leading "=" on Windows. + i = strings.Index(kv[1:], "=") + 1 + } + if i < 0 { + if kv != "" { + // The entry is not of the form "key=value" (as it is required to be). + // Leave it as-is for now. + out = append(out, kv) + } + continue + } + k := kv[:i] + if saw[k] { + continue + } + + saw[k] = true + out = append(out, kv) + } + + // Now reverse the slice to restore the original order. + for i := 0; i < len(out)/2; i++ { + j := len(out) - i - 1 + out[i], out[j] = out[j], out[i] + } + + return out +} From 76fde6ad22cdcff564e95b2aac0821d42b4bd60f Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Tue, 26 Jul 2022 07:14:30 +0000 Subject: [PATCH 02/13] Refactor --- internal/runner/runner.go | 69 +++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 903803553c..1c297746cf 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -57,46 +57,51 @@ func (r *CommandRunner) Run() (steps []*CommandStep, err error) { }() for _, step := range r.Steps { - if len(step.Command) == 0 { - err = errors.New("command is empty") + var runStep *CommandStep + runStep, err = r.runStep(step) + if err != nil { return // steps, err } + steps = append(steps, runStep) + } + return // steps, err +} - cmd := step.Command[0] - args := step.Command[1:] - // TODO: Add some kind of LD_PRELOAD protection? - env := dedupEnv(append(r.Env, step.Env...)) +func (r *CommandRunner) runStep(step *CommandStep) (*CommandStep, error) { + if len(step.Command) == 0 { + return nil, errors.New("command is empty") + } - // Set the POSIX PWD env var. - var pwd string - pwd, err = filepath.Abs(step.WorkingDir) - if err != nil { - return // steps, err - } - env = append(env, "PWD="+pwd) + cmd := step.Command[0] + args := step.Command[1:] + // TODO: Add some kind of LD_PRELOAD protection? + env := dedupEnv(append(r.Env, step.Env...)) - err = os.Chdir(pwd) - if err != nil { - return // steps, err - } + // Set the POSIX PWD env var. + pwd, err := filepath.Abs(step.WorkingDir) + if err != nil { + return nil, err + } + env = append(env, "PWD="+pwd) - // NOTE: We use syscall.Exec directly because we are executing the - // commands in sequence and we want full control over the environment. - // The stdlib exec package deduplicates env vars and so it's hard to - // know exactly what the command was run with. - // NOTE: The executed command will inherit stdout/stderr. - err = syscall.Exec(cmd, args, env) - if err != nil { - return // steps, err - } + if err = os.Chdir(pwd); err != nil { + return nil, err + } - steps = append(steps, &CommandStep{ - Command: append([]string{cmd}, args...), - Env: env, - WorkingDir: pwd, - }) + // NOTE: We use syscall.Exec directly because we are executing the + // commands in sequence and we want full control over the environment. + // The stdlib exec package deduplicates env vars and so it's hard to + // know exactly what the command was run with. + // NOTE: The executed command will inherit stdout/stderr. + if err := syscall.Exec(cmd, args, env); err != nil { + return nil, err } - return // steps, err + + return &CommandStep{ + Command: append([]string{cmd}, args...), + Env: env, + WorkingDir: pwd, + }, nil } // dedupEnv returns a copy of env with any duplicates removed, in favor of From 55241699bf8d5c666dba1747b5148e08cd6e4271 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Thu, 1 Sep 2022 04:36:32 +0000 Subject: [PATCH 03/13] Update to use os/exec Signed-off-by: Ian Lewis --- internal/runner/runner.go | 53 ++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 1c297746cf..36ba2f8818 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -1,11 +1,26 @@ +// Copyright 2022 SLSA Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package runner import ( + "context" "errors" "os" + "os/exec" "path/filepath" "strings" - "syscall" ) // CommandRunner runs commands and returns the build steps that were run. @@ -41,7 +56,7 @@ type CommandStep struct { // These are *not* the same as the runner commands. Env vars are sanitized, pwd // is changed to the absolute path, and only commands that executed // successfully are returned. -func (r *CommandRunner) Run() (steps []*CommandStep, err error) { +func (r *CommandRunner) Run(ctx context.Context) (steps []*CommandStep, err error) { var originalwd string originalwd, err = os.Getwd() if err != nil { @@ -58,7 +73,7 @@ func (r *CommandRunner) Run() (steps []*CommandStep, err error) { for _, step := range r.Steps { var runStep *CommandStep - runStep, err = r.runStep(step) + runStep, err = r.runStep(ctx, step) if err != nil { return // steps, err } @@ -67,15 +82,17 @@ func (r *CommandRunner) Run() (steps []*CommandStep, err error) { return // steps, err } -func (r *CommandRunner) runStep(step *CommandStep) (*CommandStep, error) { +func (r *CommandRunner) runStep(ctx context.Context, step *CommandStep) (*CommandStep, error) { if len(step.Command) == 0 { return nil, errors.New("command is empty") } - cmd := step.Command[0] + name := step.Command[0] args := step.Command[1:] + // TODO: Add some kind of LD_PRELOAD protection? - env := dedupEnv(append(r.Env, step.Env...)) + env := make([]string, len(step.Env)) + copy(env, step.Env) // Set the POSIX PWD env var. pwd, err := filepath.Abs(step.WorkingDir) @@ -84,21 +101,23 @@ func (r *CommandRunner) runStep(step *CommandStep) (*CommandStep, error) { } env = append(env, "PWD="+pwd) - if err = os.Chdir(pwd); err != nil { - return nil, err - } + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = pwd + cmd.Env = env + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + // TODO(https://github.com/slsa-framework/slsa-github-generator/issues/782): Update to Go 1.19. + // Get the environment that will be used as currently configured. Environ + // is needed to capture the actual environment used. + // env = cmd.Environ() - // NOTE: We use syscall.Exec directly because we are executing the - // commands in sequence and we want full control over the environment. - // The stdlib exec package deduplicates env vars and so it's hard to - // know exactly what the command was run with. - // NOTE: The executed command will inherit stdout/stderr. - if err := syscall.Exec(cmd, args, env); err != nil { + if err := cmd.Run(); err != nil { return nil, err } return &CommandStep{ - Command: append([]string{cmd}, args...), + Command: append([]string{name}, args...), Env: env, WorkingDir: pwd, }, nil @@ -107,7 +126,7 @@ func (r *CommandRunner) runStep(step *CommandStep) (*CommandStep, error) { // dedupEnv returns a copy of env with any duplicates removed, in favor of // later values. // Items not of the normal environment "key=value" form are preserved unchanged. -// NOTE: adapted from the stdlib exec package. +// NOTE: adapted from the stdlib os/exec package. func dedupEnv(env []string) []string { // Construct the output in reverse order, to preserve the // last occurrence of each key. From fcfbc30b217fa85e6bbbfadce96c39f53b134081 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Thu, 1 Sep 2022 05:26:16 +0000 Subject: [PATCH 04/13] Remove dedupEnv Signed-off-by: Ian Lewis --- internal/runner/runner.go | 44 --------------------------------------- 1 file changed, 44 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 36ba2f8818..8b3084e4bb 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -20,7 +20,6 @@ import ( "os" "os/exec" "path/filepath" - "strings" ) // CommandRunner runs commands and returns the build steps that were run. @@ -122,46 +121,3 @@ func (r *CommandRunner) runStep(ctx context.Context, step *CommandStep) (*Comman WorkingDir: pwd, }, nil } - -// dedupEnv returns a copy of env with any duplicates removed, in favor of -// later values. -// Items not of the normal environment "key=value" form are preserved unchanged. -// NOTE: adapted from the stdlib os/exec package. -func dedupEnv(env []string) []string { - // Construct the output in reverse order, to preserve the - // last occurrence of each key. - out := make([]string, 0, len(env)) - saw := make(map[string]bool, len(env)) - for n := len(env); n > 0; n-- { - kv := env[n-1] - - i := strings.Index(kv, "=") - if i == 0 { - // We observe in practice keys with a single leading "=" on Windows. - i = strings.Index(kv[1:], "=") + 1 - } - if i < 0 { - if kv != "" { - // The entry is not of the form "key=value" (as it is required to be). - // Leave it as-is for now. - out = append(out, kv) - } - continue - } - k := kv[:i] - if saw[k] { - continue - } - - saw[k] = true - out = append(out, kv) - } - - // Now reverse the slice to restore the original order. - for i := 0; i < len(out)/2; i++ { - j := len(out) - i - 1 - out[i], out[j] = out[j], out[i] - } - - return out -} From e840512e54cee2b181e3d1033f76dc39b1695d26 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Thu, 1 Sep 2022 05:38:52 +0000 Subject: [PATCH 05/13] Env and stdout/stderr changes Signed-off-by: Ian Lewis --- internal/runner/runner.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 8b3084e4bb..98fb5d6996 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -17,6 +17,7 @@ package runner import ( "context" "errors" + "io" "os" "os/exec" "path/filepath" @@ -29,6 +30,12 @@ type CommandRunner struct { // Steps are the steps to execute. Steps []*CommandStep + + // Stdout is the Writer used for Stdout. If nil then os.Stdout is used. + Stdout io.Writer + + // Stderr is the Writer used for Stderr. If nil then os.Stderr is used. + Stderr io.Writer } // CommandStep is a command that was executed by the builder. @@ -56,20 +63,6 @@ type CommandStep struct { // is changed to the absolute path, and only commands that executed // successfully are returned. func (r *CommandRunner) Run(ctx context.Context) (steps []*CommandStep, err error) { - var originalwd string - originalwd, err = os.Getwd() - if err != nil { - return // steps, err - } - defer func() { - // Change directories back to the original working directory but only - // return the error returned by Chdir if no other error occurred. - if chDirErr := os.Chdir(originalwd); err == nil { - // NOTE: err is returned by the function after the defer is called. - err = chDirErr - } - }() - for _, step := range r.Steps { var runStep *CommandStep runStep, err = r.runStep(ctx, step) @@ -89,9 +82,10 @@ func (r *CommandRunner) runStep(ctx context.Context, step *CommandStep) (*Comman name := step.Command[0] args := step.Command[1:] - // TODO: Add some kind of LD_PRELOAD protection? - env := make([]string, len(step.Env)) - copy(env, step.Env) + // Copy and merge the environment. + env := make([]string, len(r.Env), len(r.Env)+len(step.Env)+1) + copy(env, r.Env) + env = append(env, step.Env...) // Set the POSIX PWD env var. pwd, err := filepath.Abs(step.WorkingDir) @@ -104,7 +98,13 @@ func (r *CommandRunner) runStep(ctx context.Context, step *CommandStep) (*Comman cmd.Dir = pwd cmd.Env = env cmd.Stdout = os.Stdout + if r.Stdout != nil { + cmd.Stdout = r.Stdout + } cmd.Stderr = os.Stderr + if r.Stderr != nil { + cmd.Stderr = r.Stderr + } // TODO(https://github.com/slsa-framework/slsa-github-generator/issues/782): Update to Go 1.19. // Get the environment that will be used as currently configured. Environ From 24e6fddf63e31c84395e02c5045a005add492c01 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Thu, 1 Sep 2022 06:51:54 +0000 Subject: [PATCH 06/13] Add unit tests Signed-off-by: Ian Lewis --- internal/runner/runner_test.go | 152 +++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 internal/runner/runner_test.go diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go new file mode 100644 index 0000000000..783fad53d9 --- /dev/null +++ b/internal/runner/runner_test.go @@ -0,0 +1,152 @@ +// Copyright 2022 SLSA Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runner + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestCommandRunner_StepEnv(t *testing.T) { + out := &strings.Builder{} + r := CommandRunner{ + Env: []string{"TEST=hoge"}, + Steps: []*CommandStep{ + { + Command: []string{"bash", "-c", "echo -n $TEST"}, + // NOTE: this overrides other env var. + Env: []string{"TEST=fuga"}, + }, + }, + Stdout: out, + } + + steps, err := r.Run(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + pwd, err := os.Getwd() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + diff := cmp.Diff(steps, []*CommandStep{ + { + Command: []string{"bash", "-c", "echo -n $TEST"}, + // TODO(https://github.com/slsa-framework/slsa-github-generator/issues/782): de-duplicate env. + Env: []string{"TEST=hoge", "TEST=fuga", "PWD=" + pwd}, + WorkingDir: pwd, + }, + }) + if diff != "" { + t.Fatalf("unexpected result: %v", diff) + } + + if want, got := "fuga", out.String(); want != got { + t.Fatalf("unexpected env var value, want %q, got: %q", want, got) + } +} + +func TestCommandRunner_RunnerEnv(t *testing.T) { + out := &strings.Builder{} + r := CommandRunner{ + Env: []string{"RUNNER=hoge"}, + Steps: []*CommandStep{ + { + Command: []string{"bash", "-c", "echo -n $STEP"}, + // NOTE: this overrides other env var. + Env: []string{"STEP=fuga"}, + }, + }, + Stdout: out, + } + + steps, err := r.Run(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + pwd, err := os.Getwd() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + diff := cmp.Diff(steps, []*CommandStep{ + { + Command: []string{"bash", "-c", "echo -n $STEP"}, + Env: []string{"RUNNER=hoge", "STEP=fuga", "PWD=" + pwd}, + WorkingDir: pwd, + }, + }) + if diff != "" { + t.Fatalf("unexpected result: %v", diff) + } + + if want, got := "fuga", out.String(); want != got { + t.Fatalf("unexpected env var value, want %q, got: %q", want, got) + } +} + +func TestCommandRunner_RunnerMulti(t *testing.T) { + out := &strings.Builder{} + r := CommandRunner{ + Steps: []*CommandStep{ + { + Command: []string{"bash", "-c", "echo $STEP1"}, + Env: []string{"STEP1=hoge"}, + }, + { + Command: []string{"bash", "-c", "echo $STEP2"}, + Env: []string{"STEP2=fuga"}, + }, + }, + Stdout: out, + } + + steps, err := r.Run(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + pwd, err := os.Getwd() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + diff := cmp.Diff(steps, []*CommandStep{ + { + Command: []string{"bash", "-c", "echo $STEP1"}, + Env: []string{"STEP1=hoge", "PWD=" + pwd}, + WorkingDir: pwd, + }, + { + Command: []string{"bash", "-c", "echo $STEP2"}, + Env: []string{"STEP2=fuga", "PWD=" + pwd}, + WorkingDir: pwd, + }, + }) + if diff != "" { + t.Fatalf("unexpected result: %v", diff) + } + + if want, got := "hoge\nfuga\n", out.String(); want != got { + t.Fatalf("unexpected env var value, want %q, got: %q", want, got) + } +} From 49c6e064994da5949dfa2cca93678b0a4105284e Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Thu, 1 Sep 2022 08:11:30 +0000 Subject: [PATCH 07/13] Update go builder to use runner API Signed-off-by: Ian Lewis --- internal/builders/go/pkg/build.go | 24 ++- internal/builders/go/pkg/build_test.go | 235 +++++++++++++------------ 2 files changed, 135 insertions(+), 124 deletions(-) diff --git a/internal/builders/go/pkg/build.go b/internal/builders/go/pkg/build.go index e3e71012db..59f60fd7fc 100644 --- a/internal/builders/go/pkg/build.go +++ b/internal/builders/go/pkg/build.go @@ -15,14 +15,15 @@ package pkg import ( + "context" "errors" "fmt" "os" "path/filepath" "regexp" "strings" - "syscall" + "github.com/slsa-framework/slsa-github-generator/internal/runner" "github.com/slsa-framework/slsa-github-generator/internal/utils" ) @@ -151,17 +152,26 @@ func (b *GoBuild) Run(dry bool) error { // Generate the command. command := b.generateCommand(flags, binary) - // Change directory. - if err := os.Chdir(dir); err != nil { - return err - } - fmt.Println("dir", dir) fmt.Println("binary", binary) fmt.Println("command", command) fmt.Println("env", envs) - return syscall.Exec(b.goc, command, envs) + r := runner.CommandRunner{ + Steps: []*runner.CommandStep{ + { + Command: command, + Env: envs, + WorkingDir: dir, + }, + }, + } + + // TODO: Add steps to buildConfig + // TODO: Support dry-run in runner. + // TODO: Add a timeout? + _, err = r.Run(context.Background()) + return err } func getOutputBinaryPath(binary string) (string, error) { diff --git a/internal/builders/go/pkg/build_test.go b/internal/builders/go/pkg/build_test.go index 46086c9424..293b4d079e 100644 --- a/internal/builders/go/pkg/build_test.go +++ b/internal/builders/go/pkg/build_test.go @@ -17,6 +17,7 @@ package pkg import ( "fmt" "os" + "os/exec" "testing" "github.com/google/go-cmp/cmp" @@ -1049,120 +1050,120 @@ func asPointer(s string) *string { } // TODO(https://github.com/slsa-framework/slsa-github-generator/issues/771): reinstate test -// func TestGoBuild_Run(t *testing.T) { -// type fields struct { -// cfg *GoReleaserConfig -// goc string -// argEnv map[string]string -// } -// type args struct { -// dry bool -// } -// tests := []struct { -// name string -// fields fields -// args args -// wantErr bool -// err error -// }{ -// { -// name: "dry run valid flags", -// fields: fields{ -// cfg: &GoReleaserConfig{ -// Goos: "linux", -// Goarch: "amd64", -// Binary: "binary", -// Main: asPointer("../builders/go/main.go"), -// Dir: asPointer("../builders/go"), -// Ldflags: []string{ -// "-X main.version=1.0.0", -// }, -// }, -// }, -// args: args{ -// dry: true, -// }, -// }, -// { -// name: "non-dry valid flags", -// fields: fields{ -// cfg: &GoReleaserConfig{ -// Goos: "linux", -// Goarch: "amd64", -// Binary: "/tmp/binary", -// Main: asPointer("main.go"), -// Dir: asPointer("./testdata/go"), -// Ldflags: []string{ -// "-X main.version=1.0.0", -// }, -// }, -// }, -// args: args{ -// dry: false, -// }, -// }, -// { -// name: "slash in the binary name", -// fields: fields{ -// cfg: &GoReleaserConfig{ -// Goos: "linux", -// Goarch: "amd64", -// Binary: "tmp/binary", -// Main: asPointer("../builders/go/main.go"), -// Dir: asPointer("../builders/go"), -// }, -// }, -// args: args{ -// dry: true, -// }, -// wantErr: true, -// err: errorInvalidFilename, -// }, -// { -// name: "dry run - invalid flags", -// fields: fields{ -// cfg: &GoReleaserConfig{ -// Goos: "linux", -// Goarch: "amd64", -// Binary: "binary", -// Main: asPointer("../builders/go/main.go"), -// Dir: asPointer("../builders/go"), -// Ldflags: []string{}, -// }, -// }, -// args: args{ -// dry: true, -// }, -// wantErr: false, -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// b := &GoBuild{ -// cfg: tt.fields.cfg, -// goc: tt.fields.goc, -// argEnv: tt.fields.argEnv, -// } -// t.Setenv("OUTPUT_BINARY", tt.fields.cfg.Binary) -// // if the test is not dry run , then code has to look for golang binary -// if !tt.args.dry { -// path, err := exec.LookPath("go") -// if err != nil { -// t.Errorf("exec.LookPath: %v", err) -// } -// b.goc = path -// } -// err := b.Run(tt.args.dry) -// if err != nil != tt.wantErr { -// t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) -// } -// if tt.err != nil { -// if err == nil { -// t.Errorf("Run() error = nil, wantErr %v", tt.err) -// } else if errCmp(err, tt.err) { -// t.Errorf("Run() error = %v, wantErr %v %v", err, tt.err, cmp.Diff(err, tt.err)) -// } -// } -// }) -// } -// } +func TestGoBuild_Run(t *testing.T) { + type fields struct { + cfg *GoReleaserConfig + goc string + argEnv map[string]string + } + type args struct { + dry bool + } + tests := []struct { + name string + fields fields + args args + wantErr bool + err error + }{ + { + name: "dry run valid flags", + fields: fields{ + cfg: &GoReleaserConfig{ + Goos: "linux", + Goarch: "amd64", + Binary: "binary", + Main: asPointer("../builders/go/main.go"), + Dir: asPointer("../builders/go"), + Ldflags: []string{ + "-X main.version=1.0.0", + }, + }, + }, + args: args{ + dry: true, + }, + }, + { + name: "non-dry valid flags", + fields: fields{ + cfg: &GoReleaserConfig{ + Goos: "linux", + Goarch: "amd64", + Binary: "/tmp/binary", + Main: asPointer("main.go"), + Dir: asPointer("./testdata/go"), + Ldflags: []string{ + "-X main.version=1.0.0", + }, + }, + }, + args: args{ + dry: false, + }, + }, + { + name: "slash in the binary name", + fields: fields{ + cfg: &GoReleaserConfig{ + Goos: "linux", + Goarch: "amd64", + Binary: "tmp/binary", + Main: asPointer("../builders/go/main.go"), + Dir: asPointer("../builders/go"), + }, + }, + args: args{ + dry: true, + }, + wantErr: true, + err: errorInvalidFilename, + }, + { + name: "dry run - invalid flags", + fields: fields{ + cfg: &GoReleaserConfig{ + Goos: "linux", + Goarch: "amd64", + Binary: "binary", + Main: asPointer("../builders/go/main.go"), + Dir: asPointer("../builders/go"), + Ldflags: []string{}, + }, + }, + args: args{ + dry: true, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &GoBuild{ + cfg: tt.fields.cfg, + goc: tt.fields.goc, + argEnv: tt.fields.argEnv, + } + t.Setenv("OUTPUT_BINARY", tt.fields.cfg.Binary) + // if the test is not dry run , then code has to look for golang binary + if !tt.args.dry { + path, err := exec.LookPath("go") + if err != nil { + t.Errorf("exec.LookPath: %v", err) + } + b.goc = path + } + err := b.Run(tt.args.dry) + if err != nil != tt.wantErr { + t.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.err != nil { + if err == nil { + t.Errorf("Run() error = nil, wantErr %v", tt.err) + } else if !errCmp(err, tt.err) { + t.Errorf("Run() error = %v, wantErr %v %v", err, tt.err, cmp.Diff(err, tt.err)) + } + } + }) + } +} From 480d7232d7d7f269e17dc1ae97780fcedd054f88 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Fri, 2 Sep 2022 04:48:04 +0000 Subject: [PATCH 08/13] Remove old comment Signed-off-by: Ian Lewis --- internal/builders/go/pkg/build_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/builders/go/pkg/build_test.go b/internal/builders/go/pkg/build_test.go index 293b4d079e..30d4915616 100644 --- a/internal/builders/go/pkg/build_test.go +++ b/internal/builders/go/pkg/build_test.go @@ -1049,7 +1049,6 @@ func asPointer(s string) *string { return &s } -// TODO(https://github.com/slsa-framework/slsa-github-generator/issues/771): reinstate test func TestGoBuild_Run(t *testing.T) { type fields struct { cfg *GoReleaserConfig From 9d92c40f0a9b94f33539d1c4b31d9f8c4914bdec Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Fri, 2 Sep 2022 04:52:59 +0000 Subject: [PATCH 09/13] Add godoc Signed-off-by: Ian Lewis --- internal/builders/go/pkg/build.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/builders/go/pkg/build.go b/internal/builders/go/pkg/build.go index 59f60fd7fc..2307d29f18 100644 --- a/internal/builders/go/pkg/build.go +++ b/internal/builders/go/pkg/build.go @@ -55,6 +55,7 @@ var allowedEnvVariablePrefix = map[string]bool{ "GO": true, "CGO_": true, } +// GoBuild implements building a Go application. type GoBuild struct { cfg *GoReleaserConfig goc string @@ -62,6 +63,7 @@ type GoBuild struct { argEnv map[string]string } +// GoBuildNew returns a new GoBuild. func GoBuildNew(goc string, cfg *GoReleaserConfig) *GoBuild { c := GoBuild{ cfg: cfg, @@ -72,6 +74,7 @@ func GoBuildNew(goc string, cfg *GoReleaserConfig) *GoBuild { return &c } +// Run executes the build. func (b *GoBuild) Run(dry bool) error { // Get directory. dir, err := b.getDir() @@ -256,6 +259,7 @@ func (b *GoBuild) generateEnvVariables() ([]string, error) { return env, nil } +// SetArgEnvVariables sets static environment variables. func (b *GoBuild) SetArgEnvVariables(envs string) error { // Notes: // - I've tried running the re-usable workflow in a step From fa42941eada0ad54de2e34fb93fd34b4568f8751 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Fri, 2 Sep 2022 05:18:33 +0000 Subject: [PATCH 10/13] Add dry run to runner Signed-off-by: Ian Lewis --- internal/builders/go/main_test.go | 17 ++++++++++++++++ internal/builders/go/pkg/build.go | 30 ++++++++++++++++++++++------- internal/runner/runner.go | 32 ++++++++++++++++++++++++++----- 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/internal/builders/go/main_test.go b/internal/builders/go/main_test.go index d543929b56..ccdcbb595d 100644 --- a/internal/builders/go/main_test.go +++ b/internal/builders/go/main_test.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "regexp" "testing" @@ -24,6 +25,12 @@ func errCmp(e1, e2 error) bool { func Test_runBuild(t *testing.T) { t.Parallel() + + pwd, err := os.Getwd() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tests := []struct { subject string name string @@ -51,6 +58,7 @@ func Test_runBuild(t *testing.T) { "GOARCH=amd64", "GO111MODULE=on", "CGO_ENABLED=0", + "PWD=" + pwd, }, }, { @@ -68,6 +76,7 @@ func Test_runBuild(t *testing.T) { envs: []string{ "GOOS=linux", "GOARCH=amd64", + "PWD=" + pwd, }, }, { @@ -85,6 +94,7 @@ func Test_runBuild(t *testing.T) { envs: []string{ "GOOS=linux", "GOARCH=amd64", + "PWD=" + pwd, }, }, { @@ -102,6 +112,7 @@ func Test_runBuild(t *testing.T) { "GOARCH=amd64", "GO111MODULE=on", "CGO_ENABLED=0", + "PWD=" + pwd, }, }, { @@ -119,6 +130,7 @@ func Test_runBuild(t *testing.T) { "GOARCH=amd64", "GO111MODULE=on", "CGO_ENABLED=0", + "PWD=" + pwd, }, }, { @@ -138,6 +150,7 @@ func Test_runBuild(t *testing.T) { "GOARCH=amd64", "GO111MODULE=on", "CGO_ENABLED=0", + "PWD=" + pwd, }, }, { @@ -156,6 +169,7 @@ func Test_runBuild(t *testing.T) { "GOARCH=amd64", "GO111MODULE=on", "CGO_ENABLED=0", + "PWD=" + pwd, }, }, { @@ -174,6 +188,7 @@ func Test_runBuild(t *testing.T) { "GOARCH=amd64", "GO111MODULE=on", "CGO_ENABLED=0", + "PWD=" + pwd, }, }, { @@ -194,6 +209,7 @@ func Test_runBuild(t *testing.T) { "GOARCH=amd64", "GO111MODULE=on", "CGO_ENABLED=0", + "PWD=" + pwd, }, }, { @@ -214,6 +230,7 @@ func Test_runBuild(t *testing.T) { "GOARCH=amd64", "GO111MODULE=on", "CGO_ENABLED=0", + "PWD=" + filepath.Join(pwd, "./valid/path/"), }, workingDir: "./valid/path/", }, diff --git a/internal/builders/go/pkg/build.go b/internal/builders/go/pkg/build.go index 2307d29f18..1598812c41 100644 --- a/internal/builders/go/pkg/build.go +++ b/internal/builders/go/pkg/build.go @@ -120,24 +120,40 @@ func (b *GoBuild) Run(dry bool) error { // Generate the command. com := b.generateCommand(flags, filename) - // Share the resolved name of the binary. - fmt.Printf("::set-output name=go-binary-name::%s\n", filename) - command, err := utils.MarshalToString(com) + env, err := b.generateCommandEnvVariables() if err != nil { return err } - // Share the command used. - fmt.Printf("::set-output name=go-command::%s\n", command) - env, err := b.generateCommandEnvVariables() + r := runner.CommandRunner{ + Steps: []*runner.CommandStep{ + { + Command: com, + Env: env, + WorkingDir: dir, + }, + }, + } + + steps, err := r.Dry() if err != nil { return err } - menv, err := utils.MarshalToString(env) + menv, err := utils.MarshalToString(steps[0].Env) if err != nil { return err } + command, err := utils.MarshalToString(steps[0].Command) + if err != nil { + return err + } + + // Share the resolved name of the binary. + fmt.Printf("::set-output name=go-binary-name::%s\n", filename) + + // Share the command used. + fmt.Printf("::set-output name=go-command::%s\n", command) // Share the env variables used. fmt.Printf("::set-output name=go-env::%s\n", menv) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 98fb5d6996..189ffb0f70 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -50,7 +50,24 @@ type CommandStep struct { WorkingDir string `json:"workingDir"` } -// Run excutes a series of commands and returns the steps that were executed +// Dry returns the command steps as they would be executed by the runner +// without actually executing the commands. This allows builders to get an +// accurate set of steps in a trusted environment as executing commands will +// execute untrusted code. +func (r *CommandRunner) Dry() (steps []*CommandStep, err error) { + ctx := context.Background() + for _, step := range r.Steps { + var runStep *CommandStep + runStep, err = r.runStep(ctx, step, true) + if err != nil { + return // steps, err + } + steps = append(steps, runStep) + } + return // steps, err +} + +// Run executes a series of commands and returns the steps that were executed // successfully. Commands are run in sequence and are expected to return a zero // exit status. // @@ -65,7 +82,7 @@ type CommandStep struct { func (r *CommandRunner) Run(ctx context.Context) (steps []*CommandStep, err error) { for _, step := range r.Steps { var runStep *CommandStep - runStep, err = r.runStep(ctx, step) + runStep, err = r.runStep(ctx, step, false) if err != nil { return // steps, err } @@ -74,7 +91,10 @@ func (r *CommandRunner) Run(ctx context.Context) (steps []*CommandStep, err erro return // steps, err } -func (r *CommandRunner) runStep(ctx context.Context, step *CommandStep) (*CommandStep, error) { +// runStep runs the build step and returns the CommandStep configuration +// actually used to run the command. If dry is true then the CommandStep is +// returned without executing the command. +func (r *CommandRunner) runStep(ctx context.Context, step *CommandStep, dry bool) (*CommandStep, error) { if len(step.Command) == 0 { return nil, errors.New("command is empty") } @@ -111,8 +131,10 @@ func (r *CommandRunner) runStep(ctx context.Context, step *CommandStep) (*Comman // is needed to capture the actual environment used. // env = cmd.Environ() - if err := cmd.Run(); err != nil { - return nil, err + if !dry { + if err := cmd.Run(); err != nil { + return nil, err + } } return &CommandStep{ From 449c3786667ec0ddb8785f4014e6b4e90eaab070 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Fri, 2 Sep 2022 06:30:52 +0000 Subject: [PATCH 11/13] Remove old comments Signed-off-by: Ian Lewis --- internal/builders/go/pkg/build.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/builders/go/pkg/build.go b/internal/builders/go/pkg/build.go index 1598812c41..12e72a9167 100644 --- a/internal/builders/go/pkg/build.go +++ b/internal/builders/go/pkg/build.go @@ -186,8 +186,6 @@ func (b *GoBuild) Run(dry bool) error { }, } - // TODO: Add steps to buildConfig - // TODO: Support dry-run in runner. // TODO: Add a timeout? _, err = r.Run(context.Background()) return err From ba7db0059d657e104a3690414ea52ec8eddbbe0d Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Fri, 2 Sep 2022 06:31:08 +0000 Subject: [PATCH 12/13] Add PWD to pre-submit check Signed-off-by: Ian Lewis --- .github/workflows/scripts/pre-submit.e2e.go.default.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/pre-submit.e2e.go.default.sh b/.github/workflows/scripts/pre-submit.e2e.go.default.sh index b8c959ea67..9470e655bb 100755 --- a/.github/workflows/scripts/pre-submit.e2e.go.default.sh +++ b/.github/workflows/scripts/pre-submit.e2e.go.default.sh @@ -42,7 +42,7 @@ e2e_verify_predicate_buildConfig_step_env "0" "$ATTESTATION" "[]" e2e_verify_predicate_buildConfig_step_workingDir "0" "$ATTESTATION" "$PWD/internal/builders/go/e2e-presubmits" # Second step is the actual compilation. -e2e_verify_predicate_buildConfig_step_env "1" "$ATTESTATION" "[\"GOOS=linux\",\"GOARCH=amd64\",\"GO111MODULE=on\",\"CGO_ENABLED=0\"]" +e2e_verify_predicate_buildConfig_step_env "1" "$ATTESTATION" "[\"GOOS=linux\",\"GOARCH=amd64\",\"GO111MODULE=on\",\"CGO_ENABLED=0\", \"PWD=$PWD/internal/builders/go/e2e-presubmits\"]" e2e_verify_predicate_buildConfig_step_workingDir "1" "$ATTESTATION" "$PWD/internal/builders/go/e2e-presubmits" if [[ -n "$LDFLAGS" ]]; then From 38aebfbe28710061c5059245340944d4939051e5 Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Wed, 7 Sep 2022 01:34:40 +0000 Subject: [PATCH 13/13] Add comment Signed-off-by: Ian Lewis --- internal/builders/go/pkg/build.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/builders/go/pkg/build.go b/internal/builders/go/pkg/build.go index 12e72a9167..a9f42f82de 100644 --- a/internal/builders/go/pkg/build.go +++ b/internal/builders/go/pkg/build.go @@ -140,6 +140,8 @@ func (b *GoBuild) Run(dry bool) error { return err } + // There is a single command in steps given to the runner so we are + // assured to have only one step. menv, err := utils.MarshalToString(steps[0].Env) if err != nil { return err