local backend ignore errors in commands inbetween (#2636)

for normal posix shells we have to add the `-e ` option ... but as there
are more shells out there we have to handle this edgecases on base per
base case.

create a switch case statement in woodpecker should be fine as there is
only a finite number of shells, used in production.

also close  #2612

---
*Sponsored by Kithara Software GmbH*

go.mod

@@ -52,6 +52,7 @@ require (
 	golang.org/x/oauth2 v0.13.0
 	golang.org/x/sync v0.4.0
 	golang.org/x/term v0.13.0
+	golang.org/x/text v0.13.0
 	google.golang.org/grpc v1.59.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -140,7 +141,6 @@ require (
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/mod v0.13.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect

pipeline/backend/common/script.go

@@ -37,10 +37,3 @@ func GenerateContainerConf(commands []string) (env map[string]string, entry, cmd
 
 	return env, entry, cmd
 }
-
-func GenerateScript(commands []string) string {
-	if runtime.GOOS == "windows" {
-		return generateScriptWindows(commands)
-	}
-	return generateScriptPosix(commands)
-}

pipeline/backend/common/script_posix.go

@@ -25,18 +25,18 @@ import (
 // for a linux container using the given
 func generateScriptPosix(commands []string) string {
 	var buf bytes.Buffer
+
+	buf.WriteString(setupScript)
+
 	for _, command := range commands {
 		buf.WriteString(fmt.Sprintf(
 			traceScript,
 			shellescape.Quote(command),
 			command,
 		))
 	}
-	script := fmt.Sprintf(
-		setupScript,
-		buf.String(),
-	)
-	return script
+
+	return buf.String()
 }
 
 // setupScript is a helper script this is added to the step script to ensure
@@ -53,7 +53,6 @@ fi
 unset CI_NETRC_USERNAME
 unset CI_NETRC_PASSWORD
 unset CI_SCRIPT
-%s
 `
 
 // traceScript is a helper script that is added to the step script

pipeline/backend/common/script_posix_test.go

@@ -48,7 +48,6 @@ go build
 
 echo + 'go test'
 go test
-
 `,
 		},
 	}

pipeline/backend/local/command.go

@@ -0,0 +1,59 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package local
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/alessio/shellescape"
+)
+
+func genCmdByShell(shell string, cmds []string) (args []string, err error) {
+	script := ""
+	for _, cmd := range cmds {
+		script += fmt.Sprintf("echo %s\n%s\n", strings.TrimSpace(shellescape.Quote("+ "+cmd)), cmd)
+	}
+	script = strings.TrimSpace(script)
+
+	switch strings.TrimSuffix(strings.ToLower(shell), ".exe") {
+	case "cmd":
+		script := ""
+		for _, cmd := range cmds {
+			script += fmt.Sprintf("%s || exit 1\n", cmd)
+		}
+		cmd, err := os.CreateTemp(os.TempDir(), "*.cmd")
+		if err != nil {
+			return nil, err
+		}
+		defer cmd.Close()
+		if _, err := cmd.WriteString(script); err != nil {
+			return nil, err
+		}
+		return []string{"/c", cmd.Name()}, nil
+	case "fish":
+		script := ""
+		for _, cmd := range cmds {
+			script += fmt.Sprintf("echo %s\n%s || exit $status\n", strings.TrimSpace(shellescape.Quote("+ "+cmd)), cmd)
+		}
+		return []string{"-c", script}, nil
+	case "powershell", "pwsh":
+		return []string{"-noprofile", "-noninteractive", "-c", "$ErrorActionPreference = \"Stop\"; " + script}, nil
+	default:
+		// normal posix shells
+		return []string{"-e", "-c", script}, nil
+	}
+}

pipeline/backend/local/local.go

@@ -22,12 +22,13 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"slices"
-	"strings"
 	"sync"
 
-	"github.com/alessio/shellescape"
 	"github.com/rs/zerolog/log"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 
 	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 )
@@ -131,23 +132,27 @@ func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string
 
 // execCommands use step.Image as shell and run the commands in it
 func (e *local) execCommands(ctx context.Context, step *types.Step, state *workflowState, env []string) error {
-	// TODO: find a way to simulate commands to be exec as stdin user commands instead of generating a script and hope the shell understands
-	script := ""
-	for _, cmd := range step.Commands {
-		script += fmt.Sprintf("echo + %s\n%s\n", strings.TrimSpace(shellescape.Quote(cmd)), cmd)
+	// Prepare commands
+	args, err := genCmdByShell(step.Image, step.Commands)
+	if err != nil {
+		return fmt.Errorf("could not convert commands into args: %w", err)
 	}
-	script = strings.TrimSpace(script)
 
-	// Prepare command
 	// Use "image name" as run command (indicate shell)
-	cmd := exec.CommandContext(ctx, step.Image, "-c", script)
+	cmd := exec.CommandContext(ctx, step.Image, args...)
 	cmd.Env = env
 	cmd.Dir = state.workspaceDir
 
 	// Get output and redirect Stderr to Stdout
 	e.output, _ = cmd.StdoutPipe()
 	cmd.Stderr = cmd.Stdout
 
+	if runtime.GOOS == "windows" {
+		// we get non utf8 output from windows so just sanitize it
+		// TODO: remove hack
+		e.output = io.NopCloser(transform.NewReader(e.output, unicode.UTF8.NewDecoder().Transformer))
+	}
+
 	state.stepCMDs[step.Name] = cmd
 
 	return cmd.Start()