diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 3b08a31c0..d319bbd0a 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -35,5 +35,5 @@ brews: name: homebrew-tap folder: Formula homepage: 'https://github.com/yohamta/dagu' - description: 'A standalone Low-Code Workflow Executor that runs DAGs defined in a simple, declarative YAML format that is similar to GitHub Actions or Argo Workflows with built-in Web UI' + description: 'A No-code workflow executor that runs DAGs defined in a simple YAML format' license: "GNU General Public License v3.0" diff --git a/README.md b/README.md index 7e3539be4..e20cbdf5e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![GoDoc](https://godoc.org/github.com/yohamta/dagu?status.svg)](https://godoc.org/github.com/yohamta/dagu) ![Test](https://github.com/yohamta/dagu/actions/workflows/test.yaml/badge.svg) -**A No-code workflow executor with built-in web UI** +**A No-code DAG executor with built-in web UI** It runs [DAGs (Directed acyclic graph)](https://en.wikipedia.org/wiki/Directed_acyclic_graph) defined in a simple, declarative YAML format. @@ -24,9 +24,9 @@ It runs [DAGs (Directed acyclic graph)](https://en.wikipedia.org/wiki/Directed_a - [via GitHub Release Page](#via-github-release-page) - [️Quick start](#️quick-start) - [1. Launch the Web UI](#1-launch-the-web-ui) - - [2. Create a new workflow](#2-create-a-new-workflow) - - [3. Edit the workflow](#3-edit-the-workflow) - - [4. Execute the workflow](#4-execute-the-workflow) + - [2. Create a new DAG](#2-create-a-new-dag) + - [3. Edit the DAG](#3-edit-the-dag) + - [4. Execute the DAG](#4-execute-the-dag) - [Command Line User Interface](#command-line-user-interface) - [Web User Interface](#web-user-interface) - [YAML format](#yaml-format) @@ -51,7 +51,7 @@ It runs [DAGs (Directed acyclic graph)](https://en.wikipedia.org/wiki/Directed_a - [Where is the history data stored?](#where-is-the-history-data-stored) - [Where are the log files stored?](#where-are-the-log-files-stored) - [How long will the history data be stored?](#how-long-will-the-history-data-be-stored) - - [How can I retry a workflow from a specific task?](#how-can-i-retry-a-workflow-from-a-specific-task) + - [How can I retry a DAG from a specific task?](#how-can-i-retry-a-dag-from-a-specific-task) - [Does it provide sucheduler daemon?](#does-it-provide-sucheduler-daemon) - [How does it track running processes without DBMS?](#how-does-it-track-running-processes-without-dbms) - [License](#license) @@ -63,7 +63,7 @@ In the projects I worked on, our ETL pipeline had **many problems**. There were ## Why not existing tools, like Airflow? -There are many popular workflow engines such as Airflow, Prefect, etc. They are powerful and valuable tools, but they require writing code such as Python to run workflows. In many situations like above, there are already hundreds of thousands of existing lines of code in other languages such as shell scripts or Perl. Adding another layer of Python on top of these would make it more complicated. So we developed Dagu. It is easy-to-use and self-contained, making it ideal for smaller projects with fewer people. +There are many popular workflow engines such as Airflow, Prefect, etc. They are powerful and valuable tools, but they require writing code such as Python to run DAGs. In many situations like above, there are already hundreds of thousands of existing lines of code in other languages such as shell scripts or Perl. Adding another layer of Python on top of these would make it more complicated. So we developed Dagu. It is easy-to-use and self-contained, making it ideal for smaller projects with fewer people. ## How does it work? @@ -100,15 +100,15 @@ Download the latest binary from the [Releases page](https://github.com/yohamta/d Start the server with `dagu server` and browse to `http://127.0.0.1:8080` to explore the Web UI. -### 2. Create a new workflow +### 2. Create a new DAG -Create a workflow by clicking the `New DAG` button on the top page of the web UI. Input `example.yaml` in the dialog. +Create a DAG by clicking the `New DAG` button on the top page of the web UI. Input `example.yaml` in the dialog. -### 3. Edit the workflow +### 3. Edit the DAG -Go to the workflow detail page and click the `Edit` button in the `Config` Tab. Copy and paste from this [example YAML](https://github.com/yohamta/dagu/blob/main/examples/example.yaml) and click the `Save` button. +Go to the DAG detail page and click the `Edit` button in the `Config` Tab. Copy and paste from this [example YAML](https://github.com/yohamta/dagu/blob/main/examples/example.yaml) and click the `Save` button. -### 4. Execute the workflow +### 4. Execute the DAG You can execute the example by pressing the `Start` button. @@ -116,11 +116,11 @@ You can execute the example by pressing the `Start` button. ## Command Line User Interface -- `dagu start [--params=] ` - Runs the workflow -- `dagu status ` - Displays the current status of the workflow -- `dagu retry --req= ` - Re-runs the specified workflow run -- `dagu stop ` - Stops the workflow execution by sending TERM signals -- `dagu dry [--params=] ` - Dry-runs the workflow +- `dagu start [--params=] ` - Runs the DAG +- `dagu status ` - Displays the current status of the DAG +- `dagu retry --req= ` - Re-runs the specified DAG run +- `dagu stop ` - Stops the DAG execution by sending TERM signals +- `dagu dry [--params=] ` - Dry-runs the DAG - `dagu server` - Starts the web server for web UI - `dagu version` - Shows the current binary version @@ -128,13 +128,13 @@ You can execute the example by pressing the `Start` button. - **Dashboard**: It shows the overall status and executions timeline of the day. - ![Workflows](assets/images/ui-dashboard.png?raw=true) + ![DAGs](assets/images/ui-dashboard.png?raw=true) -- **Workflows**: It shows all workflows and the real-time status. +- **DAGs**: It shows all DAGs and the real-time status. - ![Workflows](assets/images/ui-workflows.png?raw=true) + ![DAGs](assets/images/ui-dags.png?raw=true) -- **Workflow Details**: It shows the real-time status, logs, and workflow configurations. You can edit workflow configurations on a browser. +- **DAG Details**: It shows the real-time status, logs, and DAG configurations. You can edit DAG configurations on a browser. ![Details](assets/images/ui-details.png?raw=true) @@ -142,15 +142,15 @@ You can execute the example by pressing the `Start` button. ![History](assets/images/ui-history.png?raw=true) -- **Workflow Execution Log**: It shows the detail log and standard output of each execution and steps. +- **DAG Execution Log**: It shows the detail log and standard output of each execution and steps. - ![Workflow Log](assets/images/ui-logoutput.png?raw=true) + ![DAG Log](assets/images/ui-logoutput.png?raw=true) ## YAML format ### Minimal Definition -Minimal workflow definition is as simple as follows: +Minimal DAG definition is as simple as follows: ```yaml steps: @@ -229,7 +229,7 @@ steps: ### Conditional Logic -Sometimes you have parts of a workflow that you only want to run under certain conditions. You can use the `precondition` field to add conditional branches to your workflow. +Sometimes you have parts of a DAG that you only want to run under certain conditions. You can use the `precondition` field to add conditional branches to your DAG. For example, the below task only runs on the first date of each month. @@ -242,7 +242,7 @@ steps: expected: "01" ``` -If you want the workflow to continue to the next step regardless of the step's conditional check result, you can use the `continueOn` field: +If you want the DAG to continue to the next step regardless of the step's conditional check result, you can use the `continueOn` field: ```yaml steps: @@ -279,7 +279,7 @@ steps: ### State Handlers -It is often desirable to take action when a specific event happens, for example, when a workflow fails. To achieve this, you can use `handlerOn` fields. +It is often desirable to take action when a specific event happens, for example, when a DAG fails. To achieve this, you can use `handlerOn` fields. ```yaml handlerOn: @@ -307,12 +307,13 @@ steps: ### All Available Fields -Combining these settings gives you granular control over how the workflow runs. +Combining these settings gives you granular control over how the DAG runs. ```yaml -name: all configuration # name (optional, default is filename) -description: run a DAG # description -tags: daily job # Free tags (separated by comma) +name: all configuration # Name (optional, default is filename) +description: run a DAG # Description +group: DailyJobs # Group name to organize DAGs (optional) +tags: example # Free tags (separated by comma) env: # Environment variables - LOG_DIR: ${HOME}/logs - PATH: /usr/local/bin:${PATH} @@ -433,9 +434,9 @@ It will store log files in the `DAGU__LOGS` environment variable path. The defau The default retention period for execution history is seven days. However, you can override the setting by the `histRetentionDays` field in a YAML file. -### How can I retry a workflow from a specific task? +### How can I retry a DAG from a specific task? -You can change the status of any task to a `failed` state. Then, when you retry the workflow, it will execute the failed one and any subsequent. +You can change the status of any task to a `failed` state. Then, when you retry the DAG, it will execute the failed one and any subsequent. ### Does it provide sucheduler daemon? diff --git a/admin/src/App.tsx b/admin/src/App.tsx index d9762ae91..906704e2b 100644 --- a/admin/src/App.tsx +++ b/admin/src/App.tsx @@ -4,8 +4,8 @@ import DashboardLayout from "./DashboardLayout"; import Dashboard from "./pages/Dashboard"; import View from "./pages/View"; import ViewList from "./pages/ViewList"; -import WorkflowDetail from "./pages/WorkflowDetails"; -import WorkflowList from "./pages/WorkflowList"; +import DAGDetails from "./pages/DAGDetails"; +import DAGs from "./pages/DAGs"; type Config = { title: string; @@ -22,11 +22,11 @@ function App({ config }: Props) { } /> - } /> + } /> } /> } /> - } /> - } /> + } /> + } /> diff --git a/admin/src/DashboardLayout.tsx b/admin/src/DashboardLayout.tsx index f9454235c..8bc116872 100644 --- a/admin/src/DashboardLayout.tsx +++ b/admin/src/DashboardLayout.tsx @@ -154,7 +154,7 @@ function DashboardContent({ {[ ["/", "Dashboard"], - ["/dags", "Workflows"], + ["/dags", "DAGs"], ["/views", "Views"], ].map((v) => ( any; }; -function WorkflowActions({ - status, - group, - name, - refresh = () => {}, - label = true, -}: Props) { +function DAGActions({ status, name, refresh = () => {}, label = true }: Props) { const onSubmit = React.useCallback( async ( warn: string, params: { - group: string; name: string; action: string; requestId?: string; @@ -31,7 +23,6 @@ function WorkflowActions({ return; } const form = new FormData(); - form.set("group", params.group); form.set("action", params.action); if (params.requestId) { form.set("request-id", params.requestId); @@ -71,8 +62,7 @@ function WorkflowActions({ } disabled={!buttonState["start"]} onClick={() => - onSubmit("Do you really want to start the workflow?", { - group: group, + onSubmit("Do you really want to start the DAG?", { name: name, action: "start", }) @@ -89,8 +79,7 @@ function WorkflowActions({ } disabled={!buttonState["stop"]} onClick={() => - onSubmit("Do you really want to cancel the workflow?", { - group: group, + onSubmit("Do you really want to cancel the DAG?", { name: name, action: "stop", }) @@ -110,7 +99,6 @@ function WorkflowActions({ onSubmit( `Do you really want to rerun the last execution (${status?.RequestId}) ?`, { - group: group, name: name, requestId: status?.RequestId, action: "retry", @@ -123,7 +111,7 @@ function WorkflowActions({ ); } -export default WorkflowActions; +export default DAGActions; interface ActionButtonProps { children: string; diff --git a/admin/src/components/WorkflowConfig.tsx b/admin/src/components/DAGConfig.tsx similarity index 95% rename from admin/src/components/WorkflowConfig.tsx rename to admin/src/components/DAGConfig.tsx index d530e88d8..e8d642f42 100644 --- a/admin/src/components/WorkflowConfig.tsx +++ b/admin/src/components/DAGConfig.tsx @@ -1,7 +1,7 @@ import { Box, Button, Paper, Stack, Typography } from "@mui/material"; import React from "react"; -import { GetWorkflowResponse } from "../api/Workflow"; -import { WorkflowContext } from "../contexts/WorkflowContext"; +import { GetDAGResponse } from "../api/DAG"; +import { DAGContext } from "../contexts/DAGContext"; import { Config } from "../models/Config"; import { Step } from "../models/Step"; import ConfigEditor from "./ConfigEditor"; @@ -11,10 +11,10 @@ import Graph from "./Graph"; import ConfigStepTable from "./ConfigStepTable"; type Props = { - data: GetWorkflowResponse; + data: GetDAGResponse; }; -function WorkflowConfig({ data }: Props) { +function DAGConfig({ data }: Props) { const [editing, setEditing] = React.useState(false); const [currentValue, setCurrentValue] = React.useState(data.Definition); const handlers = getHandlersFromConfig(data.DAG?.Config); @@ -22,7 +22,7 @@ function WorkflowConfig({ data }: Props) { return null; } return ( - + {(props) => data.DAG && data.DAG.Config && ( @@ -161,10 +161,10 @@ function WorkflowConfig({ data }: Props) { ) } - + ); } -export default WorkflowConfig; +export default DAGConfig; function getHandlersFromConfig(cfg?: Config) { const r: Step[] = []; diff --git a/admin/src/components/CreateWorkflowButton.tsx b/admin/src/components/DAGCreationButton.tsx similarity index 92% rename from admin/src/components/CreateWorkflowButton.tsx rename to admin/src/components/DAGCreationButton.tsx index 21acddb42..d3a7e63c7 100644 --- a/admin/src/components/CreateWorkflowButton.tsx +++ b/admin/src/components/DAGCreationButton.tsx @@ -5,7 +5,7 @@ type Props = { refresh: () => void; }; -function CreateWorkflowButton({ refresh }: Props) { +function DAGCreationButton({ refresh }: Props) { return (