From 117fd222133df730e46ad394a54f09b27adb942a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Brauer?= Date: Mon, 29 Nov 2021 16:37:36 +0000 Subject: [PATCH] wip: integrate exprparser into act Co-authored-by: Markus Wolf --- pkg/runner/expression.go | 592 ++++++++-------------------------- pkg/runner/expression_test.go | 51 +-- pkg/runner/run_context.go | 8 + 3 files changed, 157 insertions(+), 494 deletions(-) mode change 100755 => 100644 pkg/runner/run_context.go diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index 7f9e02daa5b..230df71869c 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -1,48 +1,57 @@ package runner import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "io" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - - "github.com/robertkrimen/otto" - log "github.com/sirupsen/logrus" -) - -var expressionPattern, operatorPattern *regexp.Regexp + "fmt" + "reflect" -func init() { - expressionPattern = regexp.MustCompile(`\${{\s*(.+?)\s*}}`) - operatorPattern = regexp.MustCompile("^[!=><|&]+$") -} + "github.com/nektos/act/pkg/exprparser" +) // NewExpressionEvaluator creates a new evaluator func (rc *RunContext) NewExpressionEvaluator() ExpressionEvaluator { - vm := rc.newVM() - return &expressionEvaluator{ - vm, + // todo: cleanup EvaluationEnvironment creation + job := rc.Run.Job() + strategy := make(map[string]interface{}) + if job.Strategy != nil { + strategy["fail-fast"] = job.Strategy.FailFast + strategy["max-parallel"] = job.Strategy.MaxParallel } -} -// NewExpressionEvaluator creates a new evaluator -func (sc *StepContext) NewExpressionEvaluator() ExpressionEvaluator { - vm := sc.RunContext.newVM() - configers := []func(*otto.Otto){ - sc.vmEnv(), - sc.vmInputs(), + jobs := rc.Run.Workflow.Jobs + jobNeeds := rc.Run.Job().Needs() + + using := make(map[string]map[string]map[string]string) + for _, needs := range jobNeeds { + using[needs] = map[string]map[string]string{ + "outputs": jobs[needs].Outputs, + } + } + + ee := &exprparser.EvaluationEnvironment{ + Github: rc.getGithubContext(), + Env: rc.Env, + Job: rc.getJobContext(), + // Steps: not steps on run context, + Runner: map[string]interface{}{ + "os": "Linux", + "temp": "/tmp", + "tool_cache": "/opt/hostedtoolcache", + }, + Secrets: rc.Config.Secrets, + Strategy: strategy, + Matrix: rc.Matrix, + Needs: using, + // Inputs: no inputs on run context, } - for _, configer := range configers { - configer(vm) + return expressionEvaluator{ + interpreter: exprparser.NewInterpeter(ee, exprparser.Config{}), } +} - return &expressionEvaluator{ - vm, +// NewExpressionEvaluator creates a new evaluator +func (sc *StepContext) NewExpressionEvaluator() ExpressionEvaluator { + return expressionEvaluator{ + interpreter: exprparser.NewInterpeter(&exprparser.EvaluationEnvironment{}, exprparser.Config{}), } } @@ -51,470 +60,153 @@ type ExpressionEvaluator interface { Evaluate(string) (string, bool, error) Interpolate(string) string InterpolateWithStringCheck(string) (string, bool) - Rewrite(string) string } type expressionEvaluator struct { - vm *otto.Otto + interpreter exprparser.Interpreter } -func (ee *expressionEvaluator) Evaluate(in string) (string, bool, error) { - if strings.HasPrefix(in, `secrets.`) { - in = `secrets.` + strings.ToUpper(strings.SplitN(in, `.`, 2)[1]) - } - re := ee.Rewrite(in) - if re != in { - log.Debugf("Evaluating '%s' instead of '%s'", re, in) - } - - val, err := ee.vm.Run(re) - if err != nil { - return "", false, err - } - if val.IsNull() || val.IsUndefined() { - return "", false, nil - } - valAsString, err := val.ToString() +func (ee expressionEvaluator) Evaluate(in string) (string, bool, error) { + evaluated, _, err := ee.interpreter.Evaluate(in) if err != nil { return "", false, err } - return valAsString, val.IsString(), err -} + fmt.Printf("%+v", evaluated) -func (ee *expressionEvaluator) Interpolate(in string) string { - interpolated, _ := ee.InterpolateWithStringCheck(in) - return interpolated -} - -func (ee *expressionEvaluator) InterpolateWithStringCheck(in string) (string, bool) { - errList := make([]error, 0) - - out := in - isString := false - for { - out = expressionPattern.ReplaceAllStringFunc(in, func(match string) string { - // Extract and trim the actual expression inside ${{...}} delimiters - expression := expressionPattern.ReplaceAllString(match, "$1") - - // Evaluate the expression and retrieve errors if any - evaluated, evaluatedIsString, err := ee.Evaluate(expression) - if err != nil { - errList = append(errList, err) - } - isString = evaluatedIsString - return evaluated - }) - if len(errList) > 0 { - log.Errorf("Unable to interpolate string '%s' - %v", in, errList) - break - } - if out == in { - // No replacement occurred, we're done! - break - } - in = out - } - return out, isString + return "", false, nil } -// Rewrite tries to transform any javascript property accessor into its bracket notation. -// For instance, "object.property" would become "object['property']". -func (ee *expressionEvaluator) Rewrite(in string) string { - var buf strings.Builder - r := strings.NewReader(in) - for { - c, _, err := r.ReadRune() - if err == io.EOF { - break - } - //nolint - switch { - default: - buf.WriteRune(c) - case c == '\'': - buf.WriteRune(c) - ee.advString(&buf, r) - case c == '.': - buf.WriteString("['") - ee.advPropertyName(&buf, r) - buf.WriteString("']") - } - } - return buf.String() +func (ee expressionEvaluator) Interpolate(in string) string { + str, _ := ee.InterpolateWithStringCheck(in) + return str } -func (*expressionEvaluator) advString(w *strings.Builder, r *strings.Reader) error { - for { - c, _, err := r.ReadRune() - if err != nil { - return err - } - if c != '\'' { - w.WriteRune(c) //nolint - continue - } +type interpolateState int - // Handles a escaped string: ex. 'It''s ok' - c, _, err = r.ReadRune() - if err != nil { - w.WriteString("'") //nolint - return err - } - if c != '\'' { - w.WriteString("'") //nolint - if err := r.UnreadRune(); err != nil { - return err - } - break - } - w.WriteString(`\'`) //nolint - } - return nil -} +const ( + passThrough interpolateState = iota + expressionStartDollar + expressionStartBracket1 + expressionStartBracket2 + expressionEndBracket1 + expressionEndBracket2 +) -func (*expressionEvaluator) advPropertyName(w *strings.Builder, r *strings.Reader) error { - for { - c, _, err := r.ReadRune() - if err != nil { - return err - } - if !isLetter(c) { - if err := r.UnreadRune(); err != nil { - return err - } - break - } - w.WriteRune(c) //nolint - } - return nil -} +func (ee expressionEvaluator) InterpolateWithStringCheck(in string) (string, bool) { -func isLetter(c rune) bool { - switch { - case c >= 'a' && c <= 'z': - return true - case c >= 'A' && c <= 'Z': - return true - case c >= '0' && c <= '9': - return true - case c == '_' || c == '-': - return true - default: - return false - } -} + output := "" + skip := 0 + // replacementIndex := "" -func (rc *RunContext) newVM() *otto.Otto { - configers := []func(*otto.Otto){ - vmContains, - vmStartsWith, - vmEndsWith, - vmFormat, - vmJoin, - vmToJSON, - vmFromJSON, - vmAlways, - rc.vmCancelled(), - rc.vmSuccess(), - rc.vmFailure(), - rc.vmHashFiles(), - - rc.vmGithub(), - rc.vmJob(), - rc.vmSteps(), - rc.vmRunner(), - - rc.vmSecrets(), - rc.vmStrategy(), - rc.vmMatrix(), - rc.vmEnv(), - rc.vmNeeds(), - } - vm := otto.New() - for _, configer := range configers { - configer(vm) - } - return vm -} + state := passThrough + for i, character := range in { + switch state { + case passThrough: + switch character { + case '$': + state = expressionStartDollar -func vmContains(vm *otto.Otto) { - _ = vm.Set("contains", func(searchString interface{}, searchValue string) bool { - if searchStringString, ok := searchString.(string); ok { - return strings.Contains(strings.ToLower(searchStringString), strings.ToLower(searchValue)) - } else if searchStringArray, ok := searchString.([]string); ok { - for _, s := range searchStringArray { - if strings.EqualFold(s, searchValue) { - return true - } + default: + fmt.Printf("%d output %c\n", i, character) + output += string(character) } - } - return false - }) -} -func vmStartsWith(vm *otto.Otto) { - _ = vm.Set("startsWith", func(searchString string, searchValue string) bool { - return strings.HasPrefix(strings.ToLower(searchString), strings.ToLower(searchValue)) - }) -} - -func vmEndsWith(vm *otto.Otto) { - _ = vm.Set("endsWith", func(searchString string, searchValue string) bool { - return strings.HasSuffix(strings.ToLower(searchString), strings.ToLower(searchValue)) - }) -} + case expressionStartDollar: + switch character { + case '{': + state = expressionStartBracket1 -func vmFormat(vm *otto.Otto) { - _ = vm.Set("format", func(s string, vals ...otto.Value) string { - ex := regexp.MustCompile(`(\{[0-9]+\}|\{.?|\}.?)`) - return ex.ReplaceAllStringFunc(s, func(seg string) string { - switch seg { - case "{{": - return "{" - case "}}": - return "}" default: - if len(seg) < 3 || !strings.HasPrefix(seg, "{") { - log.Errorf("The following format string is invalid: '%v'", s) - return "" - } - _i := seg[1 : len(seg)-1] - i, err := strconv.ParseInt(_i, 10, 32) - if err != nil { - log.Errorf("The following format string is invalid: '%v'. Error: %v", s, err) - return "" - } - if i >= int64(len(vals)) { - log.Errorf("The following format string references more arguments than were supplied: '%v'", s) - return "" - } - if vals[i].IsNull() || vals[i].IsUndefined() { - return "" - } - return vals[i].String() + output += "$" + output += string(character) + state = passThrough } - }) - }) -} -func vmJoin(vm *otto.Otto) { - _ = vm.Set("join", func(element interface{}, optionalElem string) string { - slist := make([]string, 0) - if elementString, ok := element.(string); ok { - slist = append(slist, elementString) - } else if elementArray, ok := element.([]string); ok { - slist = append(slist, elementArray...) - } - if optionalElem != "" { - slist = append(slist, optionalElem) - } - return strings.Join(slist, " ") - }) -} - -func vmToJSON(vm *otto.Otto) { - toJSON := func(o interface{}) string { - rtn, err := json.MarshalIndent(o, "", " ") - if err != nil { - log.Errorf("Unable to marshal: %v", err) - return "" - } - return string(rtn) - } - _ = vm.Set("toJSON", toJSON) - _ = vm.Set("toJson", toJSON) -} + case expressionStartBracket1: + switch character { + case '{': + state = expressionStartBracket2 -func vmFromJSON(vm *otto.Otto) { - fromJSON := func(str string) interface{} { - var dat interface{} - err := json.Unmarshal([]byte(str), &dat) - if err != nil { - log.Errorf("Unable to unmarshal: %v", err) - return dat - } - return dat - } - _ = vm.Set("fromJSON", fromJSON) - _ = vm.Set("fromJson", fromJSON) -} - -func (rc *RunContext) vmHashFiles() func(*otto.Otto) { - return func(vm *otto.Otto) { - _ = vm.Set("hashFiles", func(paths ...string) string { - var files []string - for i := range paths { - newFiles, err := filepath.Glob(filepath.Join(rc.Config.Workdir, paths[i])) - if err != nil { - log.Errorf("Unable to glob.Glob: %v", err) - return "" - } - files = append(files, newFiles...) + default: + output += "${" + output += string(character) + state = passThrough } - hasher := sha256.New() - for _, file := range files { - f, err := os.Open(file) + + case expressionStartBracket2: + if skip == 0 { + result, pos, err := ee.interpreter.Evaluate(in[i:]) if err != nil { - log.Errorf("Unable to os.Open: %v", err) + fmt.Printf("Failed to eval: %s\n", err) + return "", false } - if _, err := io.Copy(hasher, f); err != nil { - log.Errorf("Unable to io.Copy: %v", err) - } - if err := f.Close(); err != nil { - log.Errorf("Unable to Close file: %v", err) - } - } - return hex.EncodeToString(hasher.Sum(nil)) - }) - } -} - -func (rc *RunContext) vmSuccess() func(*otto.Otto) { - return func(vm *otto.Otto) { - _ = vm.Set("success", func() bool { - return rc.getJobContext().Status == "success" - }) - } -} -func (rc *RunContext) vmFailure() func(*otto.Otto) { - return func(vm *otto.Otto) { - _ = vm.Set("failure", func() bool { - return rc.getJobContext().Status == "failure" - }) - } -} -func vmAlways(vm *otto.Otto) { - _ = vm.Set("always", func() bool { - return true - }) -} -func (rc *RunContext) vmCancelled() func(vm *otto.Otto) { - return func(vm *otto.Otto) { - _ = vm.Set("cancelled", func() bool { - return rc.getJobContext().Status == "cancelled" - }) - } -} + output += ee.toString(result) -func (rc *RunContext) vmGithub() func(*otto.Otto) { - github := rc.getGithubContext() + fmt.Printf("read %d in '%s'\n", pos, in[i:pos]) - return func(vm *otto.Otto) { - _ = vm.Set("github", github) - } -} - -func (rc *RunContext) vmEnv() func(*otto.Otto) { - return func(vm *otto.Otto) { - env := rc.GetEnv() - log.Debugf("context env => %v", env) - _ = vm.Set("env", env) - } -} - -func (sc *StepContext) vmEnv() func(*otto.Otto) { - return func(vm *otto.Otto) { - log.Debugf("context env => %v", sc.Env) - _ = vm.Set("env", sc.Env) - } -} - -func (sc *StepContext) vmInputs() func(*otto.Otto) { - inputs := make(map[string]string) + if (pos - 1) == 0 { + state = expressionEndBracket1 + } else { + skip = i + pos - 1 + } + } else if i < skip { + // ignore + } else { + state = expressionEndBracket1 + skip = 0 + } - // Set Defaults - if sc.Action != nil { - for k, input := range sc.Action.Inputs { - inputs[k] = sc.RunContext.NewExpressionEvaluator().Interpolate(input.Default) + case expressionEndBracket1: + // todo: handle error + switch character { + case '}': + fmt.Printf("first closing bracket\n") + state = expressionEndBracket2 + } + case expressionEndBracket2: + // todo: handle error + switch character { + case '}': + fmt.Printf("second closing bracket\n") + state = passThrough + } } } - for k, v := range sc.Step.With { - inputs[k] = sc.RunContext.NewExpressionEvaluator().Interpolate(v) - } - - return func(vm *otto.Otto) { - _ = vm.Set("inputs", inputs) - } -} - -func (rc *RunContext) vmNeeds() func(*otto.Otto) { - jobs := rc.Run.Workflow.Jobs - jobNeeds := rc.Run.Job().Needs() - - using := make(map[string]map[string]map[string]string) - for _, needs := range jobNeeds { - using[needs] = map[string]map[string]string{ - "outputs": jobs[needs].Outputs, + if state != passThrough { + switch state { + case expressionStartDollar, expressionStartBracket1, expressionStartBracket2, expressionEndBracket1, expressionEndBracket2: + return "qwerydfkjökyxfd", false } } - return func(vm *otto.Otto) { - log.Debugf("context needs => %v", using) - _ = vm.Set("needs", using) - } -} + fmt.Printf("eval result of '%s' is '%s'\n", in, output) -func (rc *RunContext) vmJob() func(*otto.Otto) { - job := rc.getJobContext() - - return func(vm *otto.Otto) { - _ = vm.Set("job", job) - } + return output, true } -func (rc *RunContext) vmSteps() func(*otto.Otto) { - ctxSteps := rc.getStepsContext() +func (ee expressionEvaluator) toString(in interface{}) string { + value := reflect.ValueOf(in) - steps := make(map[string]interface{}) - for id, ctxStep := range ctxSteps { - steps[id] = map[string]interface{}{ - "conclusion": ctxStep.Conclusion.String(), - "outcome": ctxStep.Outcome.String(), - "outputs": ctxStep.Outputs, - } - } + switch value.Kind() { + case reflect.String: + return value.String() - return func(vm *otto.Otto) { - log.Debugf("context steps => %v", steps) - _ = vm.Set("steps", steps) - } -} + case reflect.Int: + return fmt.Sprint(value.Int()) -func (rc *RunContext) vmRunner() func(*otto.Otto) { - runner := map[string]interface{}{ - "os": "Linux", - "temp": "/tmp", - "tool_cache": "/opt/hostedtoolcache", - } + case reflect.Float64: + return fmt.Sprint(value.Float()) - return func(vm *otto.Otto) { - _ = vm.Set("runner", runner) - } -} + case reflect.Bool: + return fmt.Sprint(value.Bool()) -func (rc *RunContext) vmSecrets() func(*otto.Otto) { - return func(vm *otto.Otto) { - _ = vm.Set("secrets", rc.Config.Secrets) - } -} + case reflect.Invalid: + return "null" -func (rc *RunContext) vmStrategy() func(*otto.Otto) { - job := rc.Run.Job() - strategy := make(map[string]interface{}) - if job.Strategy != nil { - strategy["fail-fast"] = job.Strategy.FailFast - strategy["max-parallel"] = job.Strategy.MaxParallel - } - return func(vm *otto.Otto) { - _ = vm.Set("strategy", strategy) - } -} - -func (rc *RunContext) vmMatrix() func(*otto.Otto) { - return func(vm *otto.Otto) { - _ = vm.Set("matrix", rc.Matrix) + default: + panic(fmt.Sprintf("unimplemented %s %+v", value.Kind(), in)) } } diff --git a/pkg/runner/expression_test.go b/pkg/runner/expression_test.go index f372c97ebc8..598afecb92a 100644 --- a/pkg/runner/expression_test.go +++ b/pkg/runner/expression_test.go @@ -80,8 +80,8 @@ func TestEvaluate(t *testing.T) { errMesg string }{ {" 1 ", "1", ""}, - {"1 + 3", "4", ""}, - {"(1 + 3) * -2", "-8", ""}, + // {"1 + 3", "4", ""}, + // {"(1 + 3) * -2", "-8", ""}, {"'my text'", "my text", ""}, {"contains('my text', 'te')", "true", ""}, {"contains('my TEXT', 'te')", "true", ""}, @@ -182,6 +182,9 @@ func TestInterpolate(t *testing.T) { in string out string }{ + {" text ", " text "}, + {" $text ", " $text "}, + {" ${text} ", " ${text} "}, {" ${{1}} to ${{2}} ", " 1 to 2 "}, {" ${{ env.KEYWITHNOTHING }} ", " valuewithnothing "}, {" ${{ env.KEY-WITH-HYPHENS }} ", " value-with-hyphens "}, @@ -206,12 +209,13 @@ func TestInterpolate(t *testing.T) { {"${{ env.SOMETHING_TRUE || false }}", "true"}, {"${{ env.SOMETHING_FALSE || false }}", "false"}, {"${{ env.SOMETHING_FALSE }} && ${{ env.SOMETHING_TRUE }}", "false && true"}, + {"${{ fromJSON('{}') < 2 }}", "false"}, } updateTestExpressionWorkflow(t, tables, rc) for _, table := range tables { table := table - t.Run(table.in, func(t *testing.T) { + t.Run("interpolate", func(t *testing.T) { assertObject := assert.New(t) out := ee.Interpolate(table.in) assertObject.Equal(table.out, out, table.in) @@ -266,44 +270,3 @@ jobs: t.Fatal(err) } } - -func TestRewrite(t *testing.T) { - rc := &RunContext{ - Config: &Config{}, - Run: &model.Run{ - JobID: "job1", - Workflow: &model.Workflow{ - Jobs: map[string]*model.Job{ - "job1": {}, - }, - }, - }, - } - ee := rc.NewExpressionEvaluator() - - tables := []struct { - in string - re string - }{ - {"ecole", "ecole"}, - {"ecole.centrale", "ecole['centrale']"}, - {"ecole['centrale']", "ecole['centrale']"}, - {"ecole.centrale.paris", "ecole['centrale']['paris']"}, - {"ecole['centrale'].paris", "ecole['centrale']['paris']"}, - {"ecole.centrale['paris']", "ecole['centrale']['paris']"}, - {"ecole['centrale']['paris']", "ecole['centrale']['paris']"}, - {"ecole.centrale-paris", "ecole['centrale-paris']"}, - {"ecole['centrale-paris']", "ecole['centrale-paris']"}, - {"ecole.centrale_paris", "ecole['centrale_paris']"}, - {"ecole['centrale_paris']", "ecole['centrale_paris']"}, - } - - for _, table := range tables { - table := table - t.Run(table.in, func(t *testing.T) { - assertObject := assert.New(t) - re := ee.Rewrite(table.in) - assertObject.Equal(table.re, re, table.in) - }) - } -} diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go old mode 100755 new mode 100644 index 293fefb2a16..f0b2dd055e3 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -26,6 +26,14 @@ import ( const ActPath string = "/var/run/act" +// TODO: remove these patterns and rewrite EvalBool +var expressionPattern, operatorPattern *regexp.Regexp + +func init() { + expressionPattern = regexp.MustCompile(`\${{\s*(.+?)\s*}}`) + operatorPattern = regexp.MustCompile("^[!=><|&]+$") +} + // RunContext contains info about current job type RunContext struct { Name string