From 8825d91f1022ec526edc574de6f064f330733afd Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 8 Jun 2023 02:36:12 +0200 Subject: [PATCH 1/4] rm dublicated import --- cli/exec/exec.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/exec/exec.go b/cli/exec/exec.go index 3e50d50ef8..63523d02e1 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -30,7 +30,6 @@ import ( "github.com/woodpecker-ci/woodpecker/cli/common" "github.com/woodpecker-ci/woodpecker/pipeline" "github.com/woodpecker-ci/woodpecker/pipeline/backend" - "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" backendTypes "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler" @@ -202,7 +201,7 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error return err } - backendCtx := context.WithValue(c.Context, types.CliContext, c) + backendCtx := context.WithValue(c.Context, backendTypes.CliContext, c) backend.Init(backendCtx) engine, err := backend.FindEngine(backendCtx, c.String("backend-engine")) From bd9cf77432c457daa01d71ee38b3670d04dd0be8 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 8 Jun 2023 03:32:19 +0200 Subject: [PATCH 2/4] split --- pipeline/stepBuilder.go | 147 +++++++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 63 deletions(-) diff --git a/pipeline/stepBuilder.go b/pipeline/stepBuilder.go index 6772470a05..e83badecca 100644 --- a/pipeline/stepBuilder.go +++ b/pipeline/stepBuilder.go @@ -17,6 +17,7 @@ package pipeline import ( "fmt" + "path" "path/filepath" "strings" @@ -78,84 +79,35 @@ func (b *StepBuilder) Build() ([]*Item, error) { axes = append(axes, matrix.Axis{}) } - for _, axis := range axes { + isMatrix := len(axes) > 1 + + for i, axis := range axes { + name := SanitizePath(y.Name) + if isMatrix { + name = path.Join(name, fmt.Sprint(i)) + } + workflow := &model.Step{ UUID: uuid.New().String(), // TODO(#1784): Remove once workflows are a separate entity in database PipelineID: b.Curr.ID, PID: pidSequence, State: model.StatusPending, Environ: axis, - Name: SanitizePath(y.Name), - } - - workflowMetadata := frontend.MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Last, workflow, b.Link) - environ := b.environmentVariables(workflowMetadata, axis) - - // add global environment variables for substituting - for k, v := range b.Envs { - if _, exists := environ[k]; exists { - // don't override existing values - continue - } - environ[k] = v + Name: name, } - - // substitute vars - substituted, err := frontend.EnvVarSubst(string(y.Data), environ) + item, err := b.genItemForWorkflow(workflow, axis, string(y.Data)) if err != nil { return nil, err } - - // parse yaml pipeline - parsed, err := yaml.ParseString(substituted) - if err != nil { - return nil, &yaml.PipelineParseError{Err: err} - } - - // lint pipeline - if err := linter.New( - linter.WithTrusted(b.Repo.IsTrusted), - ).Lint(parsed); err != nil { - return nil, &yaml.PipelineParseError{Err: err} - } - - // checking if filtered. - if match, err := parsed.When.Match(workflowMetadata, true); !match && err == nil { - log.Debug().Str("pipeline", workflow.Name).Msg( - "Marked as skipped, dose not match metadata", - ) - workflow.State = model.StatusSkipped - } else if err != nil { - log.Debug().Str("pipeline", workflow.Name).Msg( - "Pipeline config could not be parsed", - ) - return nil, err - } - - ir, err := b.toInternalRepresentation(parsed, environ, workflowMetadata, workflow.ID) - if err != nil { - return nil, err - } - - if len(ir.Stages) == 0 { + if item == nil { continue } - - item := &Item{ - Workflow: workflow, - Config: ir, - Labels: parsed.Labels, - DependsOn: parsed.DependsOn, - RunsOn: parsed.RunsOn, - Platform: parsed.Platform, - } - if item.Labels == nil { - item.Labels = map[string]string{} - } - items = append(items, item) pidSequence++ } + + // TODO: add summary workflow that send status back based on workflows generated by matrix function + // depend on https://github.com/woodpecker-ci/woodpecker/issues/778 } items = filterItemsWithMissingDependencies(items) @@ -168,6 +120,75 @@ func (b *StepBuilder) Build() ([]*Item, error) { return items, nil } +func (b *StepBuilder) genItemForWorkflow(workflow *model.Step, axis matrix.Axis, data string) (*Item, error) { + workflowMetadata := frontend.MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Last, workflow, b.Link) + environ := b.environmentVariables(workflowMetadata, axis) + + // add global environment variables for substituting + for k, v := range b.Envs { + if _, exists := environ[k]; exists { + // don't override existing values + continue + } + environ[k] = v + } + + // substitute vars + substituted, err := frontend.EnvVarSubst(data, environ) + if err != nil { + return nil, err + } + + // parse yaml pipeline + parsed, err := yaml.ParseString(substituted) + if err != nil { + return nil, &yaml.PipelineParseError{Err: err} + } + + // lint pipeline + if err := linter.New( + linter.WithTrusted(b.Repo.IsTrusted), + ).Lint(parsed); err != nil { + return nil, &yaml.PipelineParseError{Err: err} + } + + // checking if filtered. + if match, err := parsed.When.Match(workflowMetadata, true); !match && err == nil { + log.Debug().Str("pipeline", workflow.Name).Msg( + "Marked as skipped, dose not match metadata", + ) + workflow.State = model.StatusSkipped + } else if err != nil { + log.Debug().Str("pipeline", workflow.Name).Msg( + "Pipeline config could not be parsed", + ) + return nil, err + } + + ir, err := b.toInternalRepresentation(parsed, environ, workflowMetadata, workflow.ID) + if err != nil { + return nil, err + } + + if len(ir.Stages) == 0 { + return nil, nil + } + + item := &Item{ + Workflow: workflow, + Config: ir, + Labels: parsed.Labels, + DependsOn: parsed.DependsOn, + RunsOn: parsed.RunsOn, + Platform: parsed.Platform, + } + if item.Labels == nil { + item.Labels = map[string]string{} + } + + return item, nil +} + func stepListContainsItemsToRun(items []*Item) bool { for i := range items { if items[i].Workflow.State == model.StatusPending { From 8f37a9c24535e1c969b5bbc729172c5d4c78a665 Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Wed, 1 Nov 2023 16:29:13 +0100 Subject: [PATCH 3/4] Use separate model field --- cmd/server/flags.go | 2 +- pipeline/stepBuilder.go | 11 ++--------- server/forge/common/status.go | 4 ++++ server/model/workflow.go | 1 + 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cmd/server/flags.go b/cmd/server/flags.go index 45631a3195..cfeb763c8a 100644 --- a/cmd/server/flags.go +++ b/cmd/server/flags.go @@ -244,7 +244,7 @@ var flags = append([]cli.Flag{ EnvVars: []string{"WOODPECKER_STATUS_CONTEXT_FORMAT"}, Name: "status-context-format", Usage: "status context format", - Value: "{{ .context }}/{{ .event }}/{{ .workflow }}", + Value: "{{ .context }}/{{ .event }}/{{ .workflow }}{{if not (eq .axis_id 0}}/{{.axis_id}}{{end}}", }, &cli.BoolFlag{ EnvVars: []string{"WOODPECKER_MIGRATIONS_ALLOW_LONG"}, diff --git a/pipeline/stepBuilder.go b/pipeline/stepBuilder.go index 1b35144eb2..c3c198b317 100644 --- a/pipeline/stepBuilder.go +++ b/pipeline/stepBuilder.go @@ -17,7 +17,6 @@ package pipeline import ( "fmt" - "path" "path/filepath" "strings" @@ -78,19 +77,13 @@ func (b *StepBuilder) Build() ([]*Item, error) { axes = append(axes, matrix.Axis{}) } - isMatrix := len(axes) > 1 - for i, axis := range axes { - name := SanitizePath(y.Name) - if isMatrix { - name = path.Join(name, fmt.Sprint(i)) - } - workflow := &model.Workflow{ PID: pidSequence, State: model.StatusPending, Environ: axis, - Name: name, + Name: SanitizePath(y.Name), + AxisID: i + 1, } item, err := b.genItemForWorkflow(workflow, axis, string(y.Data)) if err != nil { diff --git a/server/forge/common/status.go b/server/forge/common/status.go index cae222fbcc..fca6ffe326 100644 --- a/server/forge/common/status.go +++ b/server/forge/common/status.go @@ -19,6 +19,8 @@ import ( "fmt" "text/template" + "github.com/rs/zerolog/log" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/model" ) @@ -41,8 +43,10 @@ func GetPipelineStatusContext(repo *model.Repo, pipeline *model.Pipeline, workfl "workflow": workflow.Name, "owner": repo.Owner, "repo": repo.Name, + "axis_id": workflow.AxisID, }) if err != nil { + log.Error().Err(err).Msg("could not create status context") return "" } diff --git a/server/model/workflow.go b/server/model/workflow.go index edd5b0a459..2cceeee15f 100644 --- a/server/model/workflow.go +++ b/server/model/workflow.go @@ -28,6 +28,7 @@ type Workflow struct { AgentID int64 `json:"agent_id,omitempty" xorm:"workflow_agent_id"` Platform string `json:"platform,omitempty" xorm:"workflow_platform"` Environ map[string]string `json:"environ,omitempty" xorm:"json 'workflow_environ'"` + AxisID int `json:"-" xorm:"workflow_axis_id"` Children []*Step `json:"children,omitempty" xorm:"-"` } From ee8f9127da887d5411e9089d03723e9bdb30ee0f Mon Sep 17 00:00:00 2001 From: qwerty287 Date: Wed, 1 Nov 2023 16:30:47 +0100 Subject: [PATCH 4/4] fix docs --- docs/docs/30-administration/10-server-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/30-administration/10-server-config.md b/docs/docs/30-administration/10-server-config.md index 9cf2a879ac..ba5ac735d1 100644 --- a/docs/docs/30-administration/10-server-config.md +++ b/docs/docs/30-administration/10-server-config.md @@ -517,7 +517,7 @@ Context prefix Woodpecker will use to publish status messages to SCM. You probab ### `WOODPECKER_STATUS_CONTEXT_FORMAT` -> Default: `{{ .context }}/{{ .event }}/{{ .workflow }}` +> Default: `{{ .context }}/{{ .event }}/{{ .workflow }}{{if not (eq .axis_id 0}}/{{.axis_id}}{{end}}` Template for the status messages published to forges, uses [Go templates](https://pkg.go.dev/text/template) as template language. Supported variables: