Add jq executor

README.md

@@ -70,6 +70,7 @@ Dagu is a tool for scheduling and running tasks based on a directed acyclic grap
 - [Executors](#executors)
   - [Running Docker Containers](#running-docker-containers)
   - [Making HTTP Requests](#making-http-requests)
+  - [Executing jq Command](#executing-jq-command)
   - [Command Execution over SSH](#command-execution-over-ssh)
 - [Admin Configuration](#admin-configuration)
 - [Environment Variable](#environment-variable)
@@ -505,6 +506,50 @@ steps:
       }      
 ```
 
+### Executing jq Command
+
+The `jq` executor can be used to transform, query, and format JSON.
+
+Query Example:
+
+```yaml
+steps:
+  - name: run query
+    executor: jq
+    command: '{(.id): .["10"].b}'
+    script: |
+      {"id": "sample", "10": {"b": 42}}
+```
+
+output:
+```json
+{
+    "sample": 42
+}
+```
+
+Formatting JSON:
+
+```yaml
+steps:
+  - name: format json
+    executor: jq
+    script: |
+      {"id": "sample", "10": {"b": 42}}
+```
+
+output:
+```json
+{
+    "10": {
+        "b": 42
+    },
+    "id": "sample"
+}
+```
+
+The `jq` result can be used in following steps via [Output](#output) or [Stdout Redirection](#stdout-and-stderr-redirection).
+
 ### Command Execution over SSH
 
 The `ssh` executor allows us to execute commands on remote hosts over SSH.

examples/README.md

@@ -10,6 +10,8 @@
     - [How to run docker containers inside a `dagu` container](#how-to-run-docker-containers-inside-a-dagu-container)
   - [Command Execution over SSH](#command-execution-over-ssh)
   - [Making HTTP Requests](#making-http-requests)
+  - [Querying JSON with jq](#querying-json-with-jq)
+  - [Format JSON with jq](#format-json-with-jq)
   - [Sending Email Notification](#sending-email-notification)
   - [Customizing Signal on Stop](#customizing-signal-on-stop)
 - [How to contribute?](#how-to-contribute)
@@ -202,6 +204,42 @@ steps:
       }      
 ```
 
+## Querying JSON with jq
+```yaml
+steps:
+  - name: run query
+    executor: jq
+    command: '{(.id): .["10"].b}'
+    script: |
+      {"id": "sample", "10": {"b": 42}}
+```
+
+log output:
+```json
+{
+    "sample": 42
+}
+```
+
+## Format JSON with jq
+```yaml
+steps:
+  - name: format json
+    executor: jq
+    script: |
+      {"id": "sample", "10": {"b": 42}}
+```
+
+log output:
+```json
+{
+    "10": {
+        "b": 42
+    },
+    "id": "sample"
+}
+```
+
 ## Sending Email Notification
 
 Email example

examples/jq.yaml

@@ -0,0 +1,10 @@
+steps:
+  - name: run query
+    executor: jq
+    command: '{(.id): .["10"].b}'
+    script: |
+      {"id": "sample", "10": {"b": 42}}
+  - name: format json
+    executor: jq
+    script: |
+      {"id": "sample", "10": {"b": 42}}

go.mod

@@ -26,6 +26,9 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/itchyny/gojq v0.12.12 // indirect
+	github.com/itchyny/timefmt-go v0.1.5 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -38,13 +41,13 @@ require (
 require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rivo/uniseg v0.3.4 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/samber/lo v1.27.0
 	golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b
 	golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect
-	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
+	golang.org/x/sys v0.5.0
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -26,13 +26,21 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
+github.com/itchyny/gojq v0.12.12 h1:x+xGI9BXqKoJQZkr95ibpe3cdrTbY8D9lonrK433rcA=
+github.com/itchyny/gojq v0.12.12/go.mod h1:j+3sVkjxwd7A7Z5jrbKibgOLn0ZfLWkV+Awxr/pyzJE=
+github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
+github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
 github.com/jedib0t/go-pretty/v6 v6.3.6 h1:A6w2BuyPMtf7M82BGRBys9bAba2C26ZX9lrlrZ7uH6U=
 github.com/jedib0t/go-pretty/v6 v6.3.6/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -50,6 +58,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.3.4 h1:3Z3Eu6FGHZWSfNKJTOUiPatWwfc7DzJRU04jFUqJODw=
 github.com/rivo/uniseg v0.3.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -102,6 +112,8 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

internal/dag/builder.go

@@ -564,7 +564,7 @@ func assertStepDef(def *stepDef) error {
 	if def.Name == "" {
 		return fmt.Errorf("step name must be specified")
 	}
-	if def.Command == "" {
+	if def.Command == "" && def.Executor != "jq" {
 		return fmt.Errorf("step command must be specified")
 	}
 	return nil

internal/executor/jq.go

@@ -0,0 +1,73 @@
+package executor
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/itchyny/gojq"
+	"github.com/yohamta/dagu/internal/dag"
+)
+
+type JqExecutor struct {
+	stdout io.Writer
+	stderr io.Writer
+	query  string
+	input  map[string]interface{}
+}
+
+func (e *JqExecutor) SetStdout(out io.Writer) {
+	e.stdout = out
+}
+
+func (e *JqExecutor) SetStderr(out io.Writer) {
+	e.stderr = out
+}
+
+func (e *JqExecutor) Kill(sig os.Signal) error {
+	return nil
+}
+
+func (e *JqExecutor) Run() error {
+	query, err := gojq.Parse(e.query)
+	if err != nil {
+		return err
+	}
+	iter := query.Run(e.input)
+	for {
+		v, ok := iter.Next()
+		if !ok {
+			break
+		}
+		if err, ok := v.(error); ok {
+			e.stderr.Write([]byte(fmt.Sprintf("%#v", err)))
+			continue
+		}
+		val, err := json.MarshalIndent(v, "", "    ")
+		if err != nil {
+			e.stderr.Write([]byte(fmt.Sprintf("%#v", err)))
+			continue
+		}
+		e.stdout.Write(val)
+	}
+	return nil
+}
+
+func CreateJqExecutor(ctx context.Context, step *dag.Step) (Executor, error) {
+	s := os.ExpandEnv(step.Script)
+	input := map[string]interface{}{}
+	if err := json.Unmarshal([]byte(s), &input); err != nil {
+		return nil, err
+	}
+	return &JqExecutor{
+		stdout: os.Stdout,
+		input:  input,
+		query:  step.CmdWithArgs,
+	}, nil
+}
+
+func init() {
+	Register("jq", CreateJqExecutor)
+}