From ec35d43677700cf459cb27ea1197ea680ddc40af Mon Sep 17 00:00:00 2001 From: Pete Davison Date: Wed, 29 Nov 2023 19:38:12 -0600 Subject: [PATCH] feat: support negative globs (#1324) Co-authored-by: Andrey Nering --- CHANGELOG.md | 2 + docs/docs/usage.md | 31 +++++++++--- docs/static/schema.json | 23 ++++++++- internal/fingerprint/glob.go | 17 +++++-- internal/fingerprint/sources_checksum.go | 5 +- internal/fingerprint/task_test.go | 12 ++--- internal/templater/templater.go | 15 ++++++ task_test.go | 62 ++++++++++++++++-------- taskfile/glob.go | 32 ++++++++++++ taskfile/task.go | 8 +-- testdata/checksum/Taskfile.yml | 4 +- testdata/checksum/ignore_me.txt | 1 + variables.go | 4 +- watch.go | 7 ++- 14 files changed, 173 insertions(+), 50 deletions(-) create mode 100644 taskfile/glob.go create mode 100644 testdata/checksum/ignore_me.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index e762d47bbf..8f7b71aad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Added ability to exclude some files from `sources:` by using `exclude:` (#225, + #1324 by @pd93 and @andreynering). - The [Remote Taskfiles experiment](https://taskfile.dev/experiments/remote-taskfiles) now prefers remote files over cached ones by default (#1317, #1345 by @pd93). diff --git a/docs/docs/usage.md b/docs/docs/usage.md index 9ed31b3d77..45aeb60aea 100644 --- a/docs/docs/usage.md +++ b/docs/docs/usage.md @@ -642,11 +642,27 @@ tasks: - public/bundle.css ``` -`sources` and `generates` can be files or file patterns. When given, Task will +`sources` and `generates` can be files or glob patterns. When given, Task will compare the checksum of the source files to determine if it's necessary to run the task. If not, it will just print a message like `Task "js" is up to date`. -If you prefer this check to be made by the modification timestamp of the files, +`exclude:` can also be used to exclude files from fingerprinting. +Sources are evaluated in order, so `exclude:` must come after the positive +glob it is negating. + +```yaml +version: '3' + +tasks: + css: + sources: + - mysources/**/*.css + - exclude: mysources/ignoreme.css + generates: + - public/bundle.css +``` + +If you prefer these check to be made by the modification timestamp of the files, instead of its checksum (content), just set the `method` property to `timestamp`. @@ -1001,9 +1017,9 @@ This works for all types of variables. ## Looping over values -As of v3.28.0, Task allows you to loop over certain values and execute a -command for each. There are a number of ways to do this depending on the type -of value you want to loop over. +As of v3.28.0, Task allows you to loop over certain values and execute a command +for each. There are a number of ways to do this depending on the type of value +you want to loop over. ### Looping over a static list @@ -1043,9 +1059,8 @@ match that glob. Source paths will always be returned as paths relative to the task directory. If you need to convert this to an absolute path, you can use the built-in -`joinPath` function. -There are some [special variables](/api/#special-variables) that you may find -useful for this. +`joinPath` function. There are some [special variables](/api/#special-variables) +that you may find useful for this. ```yaml version: '3' diff --git a/docs/static/schema.json b/docs/static/schema.json index 548b563a51..ca5f105c95 100644 --- a/docs/static/schema.json +++ b/docs/static/schema.json @@ -88,14 +88,14 @@ "description": "A list of sources to check before running this task. Relevant for `checksum` and `timestamp` methods. Can be file paths or star globs.", "type": "array", "items": { - "type": "string" + "$ref": "#/definitions/3/glob" } }, "generates": { "description": "A list of files meant to be generated by this task. Relevant for `timestamp` method. Can be file paths or star globs.", "type": "array", "items": { - "type": "string" + "$ref": "#/definitions/3/glob" } }, "status": { @@ -446,6 +446,25 @@ } } }, + "glob": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/3/glob_obj" + } + ] + }, + "glob_obj": { + "type": "object", + "properties": { + "exclude": { + "description": "File or glob patter to exclude from the list", + "type": "string" + } + } + }, "run": { "type": "string", "enum": ["always", "once", "when_changed"] diff --git a/internal/fingerprint/glob.go b/internal/fingerprint/glob.go index ecf0e54c68..3304e6c43e 100644 --- a/internal/fingerprint/glob.go +++ b/internal/fingerprint/glob.go @@ -8,16 +8,25 @@ import ( "github.com/go-task/task/v3/internal/execext" "github.com/go-task/task/v3/internal/filepathext" + "github.com/go-task/task/v3/taskfile" ) -func Globs(dir string, globs []string) ([]string, error) { - files := make([]string, 0) +func Globs(dir string, globs []*taskfile.Glob) ([]string, error) { + fileMap := make(map[string]bool) for _, g := range globs { - f, err := Glob(dir, g) + matches, err := Glob(dir, g.Glob) if err != nil { continue } - files = append(files, f...) + for _, match := range matches { + fileMap[match] = !g.Negate + } + } + files := make([]string, 0) + for file, includePath := range fileMap { + if includePath { + files = append(files, file) + } } sort.Strings(files) return files, nil diff --git a/internal/fingerprint/sources_checksum.go b/internal/fingerprint/sources_checksum.go index 71c3db68cd..3cb4bb7856 100644 --- a/internal/fingerprint/sources_checksum.go +++ b/internal/fingerprint/sources_checksum.go @@ -53,7 +53,10 @@ func (checker *ChecksumChecker) IsUpToDate(t *taskfile.Task) (bool, error) { if len(t.Generates) > 0 { // For each specified 'generates' field, check whether the files actually exist for _, g := range t.Generates { - generates, err := Glob(t.Dir, g) + if g.Negate { + continue + } + generates, err := Glob(t.Dir, g.Glob) if os.IsNotExist(err) { return false, nil } diff --git a/internal/fingerprint/task_test.go b/internal/fingerprint/task_test.go index 8bbff8364e..035b05934e 100644 --- a/internal/fingerprint/task_test.go +++ b/internal/fingerprint/task_test.go @@ -47,7 +47,7 @@ func TestIsTaskUpToDate(t *testing.T) { name: "expect TRUE when no status is defined and sources are up-to-date", task: &taskfile.Task{ Status: nil, - Sources: []string{"sources"}, + Sources: []*taskfile.Glob{{Glob: "sources"}}, }, setupMockStatusChecker: nil, setupMockSourcesChecker: func(m *mocks.SourcesCheckable) { @@ -59,7 +59,7 @@ func TestIsTaskUpToDate(t *testing.T) { name: "expect FALSE when no status is defined and sources are NOT up-to-date", task: &taskfile.Task{ Status: nil, - Sources: []string{"sources"}, + Sources: []*taskfile.Glob{{Glob: "sources"}}, }, setupMockStatusChecker: nil, setupMockSourcesChecker: func(m *mocks.SourcesCheckable) { @@ -83,7 +83,7 @@ func TestIsTaskUpToDate(t *testing.T) { name: "expect TRUE when status and sources are up-to-date", task: &taskfile.Task{ Status: []string{"status"}, - Sources: []string{"sources"}, + Sources: []*taskfile.Glob{{Glob: "sources"}}, }, setupMockStatusChecker: func(m *mocks.StatusCheckable) { m.EXPECT().IsUpToDate(mock.Anything, mock.Anything).Return(true, nil) @@ -97,7 +97,7 @@ func TestIsTaskUpToDate(t *testing.T) { name: "expect FALSE when status is up-to-date, but sources are NOT up-to-date", task: &taskfile.Task{ Status: []string{"status"}, - Sources: []string{"sources"}, + Sources: []*taskfile.Glob{{Glob: "sources"}}, }, setupMockStatusChecker: func(m *mocks.StatusCheckable) { m.EXPECT().IsUpToDate(mock.Anything, mock.Anything).Return(true, nil) @@ -123,7 +123,7 @@ func TestIsTaskUpToDate(t *testing.T) { name: "expect FALSE when status is NOT up-to-date, but sources are up-to-date", task: &taskfile.Task{ Status: []string{"status"}, - Sources: []string{"sources"}, + Sources: []*taskfile.Glob{{Glob: "sources"}}, }, setupMockStatusChecker: func(m *mocks.StatusCheckable) { m.EXPECT().IsUpToDate(mock.Anything, mock.Anything).Return(false, nil) @@ -137,7 +137,7 @@ func TestIsTaskUpToDate(t *testing.T) { name: "expect FALSE when status and sources are NOT up-to-date", task: &taskfile.Task{ Status: []string{"status"}, - Sources: []string{"sources"}, + Sources: []*taskfile.Glob{{Glob: "sources"}}, }, setupMockStatusChecker: func(m *mocks.StatusCheckable) { m.EXPECT().IsUpToDate(mock.Anything, mock.Anything).Return(false, nil) diff --git a/internal/templater/templater.go b/internal/templater/templater.go index c5486281e1..2b8213b373 100644 --- a/internal/templater/templater.go +++ b/internal/templater/templater.go @@ -80,6 +80,21 @@ func (r *Templater) ReplaceSlice(strs []string) []string { return new } +func (r *Templater) ReplaceGlobs(globs []*taskfile.Glob) []*taskfile.Glob { + if r.err != nil || len(globs) == 0 { + return nil + } + + new := make([]*taskfile.Glob, len(globs)) + for i, g := range globs { + new[i] = &taskfile.Glob{ + Glob: r.Replace(g.Glob), + Negate: g.Negate, + } + } + return new +} + func (r *Templater) ReplaceVars(vars *taskfile.Vars) *taskfile.Vars { return r.replaceVars(vars, nil) } diff --git a/task_test.go b/task_test.go index 43df8bf26d..66b8917179 100644 --- a/task_test.go +++ b/task_test.go @@ -1891,27 +1891,47 @@ func TestEvaluateSymlinksInPaths(t *testing.T) { Stderr: &buff, Silent: false, } - require.NoError(t, e.Setup()) - err := e.Run(context.Background(), taskfile.Call{Task: "default"}) - require.NoError(t, err) - assert.NotEqual(t, `task: Task "default" is up to date`, strings.TrimSpace(buff.String())) - buff.Reset() - err = e.Run(context.Background(), taskfile.Call{Task: "test-sym"}) - require.NoError(t, err) - assert.NotEqual(t, `task: Task "test-sym" is up to date`, strings.TrimSpace(buff.String())) - buff.Reset() - err = e.Run(context.Background(), taskfile.Call{Task: "default"}) - require.NoError(t, err) - assert.NotEqual(t, `task: Task "default" is up to date`, strings.TrimSpace(buff.String())) - buff.Reset() - err = e.Run(context.Background(), taskfile.Call{Task: "default"}) - require.NoError(t, err) - assert.Equal(t, `task: Task "default" is up to date`, strings.TrimSpace(buff.String())) - buff.Reset() - err = e.Run(context.Background(), taskfile.Call{Task: "reset"}) - require.NoError(t, err) - buff.Reset() - err = os.RemoveAll(dir + "/.task") + tests := []struct { + name string + task string + expected string + }{ + { + name: "default (1)", + task: "default", + expected: "task: [default] echo \"some job\"\nsome job", + }, + { + name: "test-sym (1)", + task: "test-sym", + expected: "task: [test-sym] echo \"shared file source changed\" > src/shared/b", + }, + { + name: "default (2)", + task: "default", + expected: "task: [default] echo \"some job\"\nsome job", + }, + { + name: "default (3)", + task: "default", + expected: `task: Task "default" is up to date`, + }, + { + name: "reset", + task: "reset", + expected: "task: [reset] echo \"shared file source\" > src/shared/b\ntask: [reset] echo \"file source\" > src/a", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.NoError(t, e.Setup()) + err := e.Run(context.Background(), taskfile.Call{Task: test.task}) + require.NoError(t, err) + assert.Equal(t, test.expected, strings.TrimSpace(buff.String())) + buff.Reset() + }) + } + err := os.RemoveAll(dir + "/.task") require.NoError(t, err) } diff --git a/taskfile/glob.go b/taskfile/glob.go new file mode 100644 index 0000000000..051dfd8e4b --- /dev/null +++ b/taskfile/glob.go @@ -0,0 +1,32 @@ +package taskfile + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +type Glob struct { + Glob string + Negate bool +} + +func (g *Glob) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + g.Glob = node.Value + return nil + case yaml.MappingNode: + var glob struct { + Exclude string + } + if err := node.Decode(&glob); err != nil { + return err + } + g.Glob = glob.Exclude + g.Negate = true + return nil + default: + return fmt.Errorf("yaml: line %d: cannot unmarshal %s into task", node.Line, node.ShortTag()) + } +} diff --git a/taskfile/task.go b/taskfile/task.go index 78239788ba..256359f683 100644 --- a/taskfile/task.go +++ b/taskfile/task.go @@ -19,8 +19,8 @@ type Task struct { Summary string Requires *Requires Aliases []string - Sources []string - Generates []string + Sources []*Glob + Generates []*Glob Status []string Preconditions []*Precondition Dir string @@ -83,8 +83,8 @@ func (t *Task) UnmarshalYAML(node *yaml.Node) error { Prompt string Summary string Aliases []string - Sources []string - Generates []string + Sources []*Glob + Generates []*Glob Status []string Preconditions []*Precondition Dir string diff --git a/testdata/checksum/Taskfile.yml b/testdata/checksum/Taskfile.yml index dc26a77dfc..9a34ea51f8 100644 --- a/testdata/checksum/Taskfile.yml +++ b/testdata/checksum/Taskfile.yml @@ -6,7 +6,9 @@ tasks: - cp ./source.txt ./generated.txt sources: - ./**/glob-with-inexistent-file.txt - - ./source.txt + - ./*.txt + - exclude: ./ignore_me.txt + - exclude: ./generated.txt generates: - ./generated.txt method: checksum diff --git a/testdata/checksum/ignore_me.txt b/testdata/checksum/ignore_me.txt new file mode 100644 index 0000000000..14e86f3a45 --- /dev/null +++ b/testdata/checksum/ignore_me.txt @@ -0,0 +1 @@ +plz ignore me diff --git a/variables.go b/variables.go index 005ad57dde..3237652fdd 100644 --- a/variables.go +++ b/variables.go @@ -50,8 +50,8 @@ func (e *Executor) compiledTask(call taskfile.Call, evaluateShVars bool) (*taskf Prompt: r.Replace(origTask.Prompt), Summary: r.Replace(origTask.Summary), Aliases: origTask.Aliases, - Sources: r.ReplaceSlice(origTask.Sources), - Generates: r.ReplaceSlice(origTask.Generates), + Sources: r.ReplaceGlobs(origTask.Sources), + Generates: r.ReplaceGlobs(origTask.Generates), Dir: r.Replace(origTask.Dir), Set: origTask.Set, Shopt: origTask.Shopt, diff --git a/watch.go b/watch.go index 95fbb435c7..702a75822c 100644 --- a/watch.go +++ b/watch.go @@ -142,7 +142,12 @@ func (e *Executor) registerWatchedFiles(w *watcher.Watcher, calls ...taskfile.Ca } } - for _, s := range task.Sources { + globs, err := fingerprint.Globs(task.Dir, task.Sources) + if err != nil { + return err + } + + for _, s := range globs { files, err := fingerprint.Glob(task.Dir, s) if err != nil { return fmt.Errorf("task: %s: %w", s, err)