From 5b3bba726db3173f497ef1def445624f60b6e08f Mon Sep 17 00:00:00 2001 From: 6543 Date: Sat, 28 Oct 2023 13:38:47 +0200 Subject: [PATCH] local backend ignore errors in commands inbetween (#2636) for normal posix shells we have to add the `-e ` option ... but as there are more shells out there we have to handle this edgecases on base per base case. create a switch case statement in woodpecker should be fine as there is only a finite number of shells, used in production. also close #2612 --- *Sponsored by Kithara Software GmbH* --- go.mod | 2 +- pipeline/backend/common/script.go | 7 --- pipeline/backend/common/script_posix.go | 11 ++-- pipeline/backend/common/script_posix_test.go | 1 - pipeline/backend/local/command.go | 59 ++++++++++++++++++++ pipeline/backend/local/local.go | 23 +++++--- 6 files changed, 79 insertions(+), 24 deletions(-) create mode 100644 pipeline/backend/local/command.go diff --git a/go.mod b/go.mod index 6832fce756..fad30e054f 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( golang.org/x/oauth2 v0.13.0 golang.org/x/sync v0.4.0 golang.org/x/term v0.13.0 + golang.org/x/text v0.13.0 google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 @@ -140,7 +141,6 @@ require ( golang.org/x/arch v0.3.0 // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.14.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/pipeline/backend/common/script.go b/pipeline/backend/common/script.go index e17d29870c..8e955b19f9 100644 --- a/pipeline/backend/common/script.go +++ b/pipeline/backend/common/script.go @@ -37,10 +37,3 @@ func GenerateContainerConf(commands []string) (env map[string]string, entry, cmd return env, entry, cmd } - -func GenerateScript(commands []string) string { - if runtime.GOOS == "windows" { - return generateScriptWindows(commands) - } - return generateScriptPosix(commands) -} diff --git a/pipeline/backend/common/script_posix.go b/pipeline/backend/common/script_posix.go index e8d21beb19..86ff0a0c1a 100644 --- a/pipeline/backend/common/script_posix.go +++ b/pipeline/backend/common/script_posix.go @@ -25,6 +25,9 @@ import ( // for a linux container using the given func generateScriptPosix(commands []string) string { var buf bytes.Buffer + + buf.WriteString(setupScript) + for _, command := range commands { buf.WriteString(fmt.Sprintf( traceScript, @@ -32,11 +35,8 @@ func generateScriptPosix(commands []string) string { command, )) } - script := fmt.Sprintf( - setupScript, - buf.String(), - ) - return script + + return buf.String() } // setupScript is a helper script this is added to the step script to ensure @@ -53,7 +53,6 @@ fi unset CI_NETRC_USERNAME unset CI_NETRC_PASSWORD unset CI_SCRIPT -%s ` // traceScript is a helper script that is added to the step script diff --git a/pipeline/backend/common/script_posix_test.go b/pipeline/backend/common/script_posix_test.go index 0129666d47..497932ed82 100644 --- a/pipeline/backend/common/script_posix_test.go +++ b/pipeline/backend/common/script_posix_test.go @@ -48,7 +48,6 @@ go build echo + 'go test' go test - `, }, } diff --git a/pipeline/backend/local/command.go b/pipeline/backend/local/command.go new file mode 100644 index 0000000000..e2b2a7eba1 --- /dev/null +++ b/pipeline/backend/local/command.go @@ -0,0 +1,59 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +import ( + "fmt" + "os" + "strings" + + "github.com/alessio/shellescape" +) + +func genCmdByShell(shell string, cmds []string) (args []string, err error) { + script := "" + for _, cmd := range cmds { + script += fmt.Sprintf("echo %s\n%s\n", strings.TrimSpace(shellescape.Quote("+ "+cmd)), cmd) + } + script = strings.TrimSpace(script) + + switch strings.TrimSuffix(strings.ToLower(shell), ".exe") { + case "cmd": + script := "" + for _, cmd := range cmds { + script += fmt.Sprintf("%s || exit 1\n", cmd) + } + cmd, err := os.CreateTemp(os.TempDir(), "*.cmd") + if err != nil { + return nil, err + } + defer cmd.Close() + if _, err := cmd.WriteString(script); err != nil { + return nil, err + } + return []string{"/c", cmd.Name()}, nil + case "fish": + script := "" + for _, cmd := range cmds { + script += fmt.Sprintf("echo %s\n%s || exit $status\n", strings.TrimSpace(shellescape.Quote("+ "+cmd)), cmd) + } + return []string{"-c", script}, nil + case "powershell", "pwsh": + return []string{"-noprofile", "-noninteractive", "-c", "$ErrorActionPreference = \"Stop\"; " + script}, nil + default: + // normal posix shells + return []string{"-e", "-c", script}, nil + } +} diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index 37169f818f..f2a68eb7d0 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -22,12 +22,13 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "slices" - "strings" "sync" - "github.com/alessio/shellescape" "github.com/rs/zerolog/log" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" ) @@ -131,16 +132,14 @@ func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string // execCommands use step.Image as shell and run the commands in it func (e *local) execCommands(ctx context.Context, step *types.Step, state *workflowState, env []string) error { - // TODO: find a way to simulate commands to be exec as stdin user commands instead of generating a script and hope the shell understands - script := "" - for _, cmd := range step.Commands { - script += fmt.Sprintf("echo + %s\n%s\n", strings.TrimSpace(shellescape.Quote(cmd)), cmd) + // Prepare commands + args, err := genCmdByShell(step.Image, step.Commands) + if err != nil { + return fmt.Errorf("could not convert commands into args: %w", err) } - script = strings.TrimSpace(script) - // Prepare command // Use "image name" as run command (indicate shell) - cmd := exec.CommandContext(ctx, step.Image, "-c", script) + cmd := exec.CommandContext(ctx, step.Image, args...) cmd.Env = env cmd.Dir = state.workspaceDir @@ -148,6 +147,12 @@ func (e *local) execCommands(ctx context.Context, step *types.Step, state *workf e.output, _ = cmd.StdoutPipe() cmd.Stderr = cmd.Stdout + if runtime.GOOS == "windows" { + // we get non utf8 output from windows so just sanitize it + // TODO: remove hack + e.output = io.NopCloser(transform.NewReader(e.output, unicode.UTF8.NewDecoder().Transformer)) + } + state.stepCMDs[step.Name] = cmd return cmd.Start()