feat(config): support workflow merge (#478)

Author: tusharmath

Cargo.lock

@@ -81,6 +81,7 @@ url = { version = "2.5.4", features = ["serde"] }
 uuid = { version = "1.11.0", features = ["v4", "fast-rng", "macro-diagnostics", "serde"] }
 whoami = "1.5.2"
 blake3 = "1.6.1"
+merge = {version = "0.1", features = ["derive"]}
 
 # Internal crates
 forge_api = { path = "crates/forge_api" }

Cargo.toml

@@ -42,17 +42,18 @@ Forge is a comprehensive coding agent that integrates AI capabilities with your
     - [ACT Mode (Default)](#act-mode-default)
     - [PLAN Mode](#plan-mode)
   - [Application Logs](#application-logs)
+- [Provider Configuration](#provider-configuration)
+  - [Supported Providers](#supported-providers)
+  - [Custom Provider URLs](#custom-provider-urls)
 - [Custom Workflows and Multi-Agent Systems](#custom-workflows-and-multi-agent-systems)
   - [Creating Custom Workflows](#creating-custom-workflows)
+  - [Configuration Loading and Precedence](#configuration-loading-and-precedence)
   - [Workflow Configuration](#workflow-configuration)
     - [Event System](#event-system)
     - [Agent Tools](#agent-tools)
     - [Agent Configuration Options](#agent-configuration-options)
     - [Built-in Templates](#built-in-templates)
     - [Example Workflow Configuration](#example-workflow-configuration)
-- [Provider Configuration](#provider-configuration)
-  - [Supported Providers](#supported-providers)
-  - [Custom Provider URLs](#custom-provider-urls)
 - [Why Shell?](#why-shell)
 - [Community](#community)
 - [Support Us](#support-us)
@@ -309,6 +310,20 @@ You can configure your own workflows by creating a YAML file and pointing to it
 forge -w /path/to/your/workflow.yaml
 ```
 
+### Configuration Loading and Precedence
+
+Forge loads workflow configurations using the following precedence rules:
+
+1. **Explicit Path**: When a path is provided with the `-w` flag, Forge loads that configuration directly without any merging
+2. **Project Configuration**: If no explicit path is provided, Forge looks for `forge.yaml` in the current directory
+3. **Default Configuration**: An embedded default configuration is always available as a fallback
+
+When a project configuration exists in the current directory, Forge creates a merged configuration where:
+- Project settings in `forge.yaml` take precedence over default settings
+- Any settings not specified in the project configuration inherit from defaults
+
+This approach allows you to customize only the parts of the configuration you need while inheriting sensible defaults for everything else.
+
 ### Workflow Configuration
 
 A workflow consists of agents connected via events. Each agent has specific capabilities and can perform designated tasks.

README.md

@@ -14,6 +14,8 @@ forge_infra.workspace = true
 forge_snaps.workspace = true
 serde_yaml.workspace = true
 serde_json.workspace = true
+merge.workspace = true
+bytes.workspace = true
 
 [dev-dependencies]
 tempfile.workspace = true

crates/forge_api/Cargo.toml

@@ -4,10 +4,21 @@ use std::sync::Arc;
 use anyhow::Context;
 use forge_app::{FsReadService, Infrastructure};
 use forge_domain::Workflow;
+use merge::Merge;
 
 // Default forge.yaml content embedded in the binary
 const DEFAULT_FORGE_WORKFLOW: &str = include_str!("../../../forge.default.yaml");
 
+/// Represents the possible sources of a workflow configuration
+enum WorkflowSource<'a> {
+    /// Explicitly provided path
+    ExplicitPath(&'a Path),
+    /// Default configuration embedded in the binary
+    Default,
+    /// Project-specific configuration in the current directory
+    ProjectConfig,
+}
+
 /// A workflow loader to load the workflow from the given path.
 /// It also resolves the internal paths specified in the workflow.
 pub struct ForgeLoaderService<F>(Arc<F>);
@@ -19,31 +30,66 @@ impl<F> ForgeLoaderService<F> {
 }
 
 impl<F: Infrastructure> ForgeLoaderService<F> {
-    /// loads the workflow from the given path.
-    /// Loads the workflow from the given path if provided, otherwise tries to
-    /// read from current directory's forge.yaml, and falls back to embedded
-    /// default if neither exists.
+    /// Loads the workflow from the given path.
+    /// If a path is provided, uses that workflow directly without merging.
+    /// If no path is provided:
+    ///   - Loads from current directory's forge.yaml merged with defaults (if
+    ///     forge.yaml exists)
+    ///   - Falls back to embedded default if forge.yaml doesn't exist
+    ///
+    /// When merging, the project's forge.yaml values take precedence over
+    /// defaults.
     pub async fn load(&self, path: Option<&Path>) -> anyhow::Result<Workflow> {
-        let content = match path {
-            Some(path) => String::from_utf8(self.0.file_read_service().read(path).await?.to_vec())?,
-            None => {
-                let current_dir_config = Path::new("forge.yaml");
-                if current_dir_config.exists() {
-                    String::from_utf8(
-                        self.0
-                            .file_read_service()
-                            .read(current_dir_config)
-                            .await?
-                            .to_vec(),
-                    )?
-                } else {
-                    DEFAULT_FORGE_WORKFLOW.to_string()
-                }
-            }
+        // Determine the workflow source
+        let source = match path {
+            Some(path) => WorkflowSource::ExplicitPath(path),
+            None if Path::new("forge.yaml").exists() => WorkflowSource::ProjectConfig,
+            None => WorkflowSource::Default,
         };
 
-        let workflow: Workflow =
-            serde_yaml::from_str(&content).with_context(|| "Failed to parse workflow")?;
+        // Load the workflow based on its source
+        match source {
+            WorkflowSource::ExplicitPath(path) => self.load_from_explicit_path(path).await,
+            WorkflowSource::Default => self.load_default_workflow(),
+            WorkflowSource::ProjectConfig => self.load_with_project_config().await,
+        }
+    }
+
+    /// Loads a workflow from a specific file path
+    async fn load_from_explicit_path(&self, path: &Path) -> anyhow::Result<Workflow> {
+        let content = String::from_utf8(self.0.file_read_service().read(path).await?.to_vec())?;
+        let workflow: Workflow = serde_yaml::from_str(&content)
+            .with_context(|| format!("Failed to parse workflow from {}", path.display()))?;
+        Ok(workflow)
+    }
+
+    /// Loads the default workflow from embedded content
+    fn load_default_workflow(&self) -> anyhow::Result<Workflow> {
+        let workflow: Workflow = serde_yaml::from_str(DEFAULT_FORGE_WORKFLOW)
+            .with_context(|| "Failed to parse default workflow")?;
         Ok(workflow)
     }
+
+    /// Loads workflow by merging project config with default workflow
+    async fn load_with_project_config(&self) -> anyhow::Result<Workflow> {
+        let default_workflow = self.load_default_workflow()?;
+        let project_path = Path::new("forge.yaml");
+
+        let project_content = String::from_utf8(
+            self.0
+                .file_read_service()
+                .read(project_path)
+                .await?
+                .to_vec(),
+        )?;
+
+        let project_workflow: Workflow = serde_yaml::from_str(&project_content)
+            .with_context(|| "Failed to parse project workflow")?;
+
+        // Merge workflows with project taking precedence
+        let mut merged_workflow = default_workflow;
+        merged_workflow.merge(project_workflow);
+
+        Ok(merged_workflow)
+    }
 }

crates/forge_api/src/loader.rs

@@ -29,13 +29,13 @@ impl ForgeProviderService {
 impl ProviderService for ForgeProviderService {
     async fn chat(
         &self,
-        model_id: &ModelId,
+        model: &ModelId,
         request: ChatContext,
     ) -> ResultStream<ChatCompletionMessage, anyhow::Error> {
         self.client
-            .chat(model_id, request)
+            .chat(model, request)
             .await
-            .with_context(|| format!("Failed to chat with model: {}", model_id))
+            .with_context(|| format!("Failed to chat with model: {}", model))
     }
 
     async fn models(&self) -> Result<Vec<Model>> {

crates/forge_app/src/provider.rs

@@ -28,10 +28,6 @@ impl Default for Fetch {
     }
 }
 
-fn default_max_length() -> Option<usize> {
-    Some(5000)
-}
-
 fn default_start_index() -> Option<usize> {
     Some(0)
 }
@@ -45,7 +41,6 @@ pub struct FetchInput {
     /// URL to fetch
     url: String,
     /// Maximum number of characters to return (default: 40000)
-    #[serde(default = "default_max_length")]
     max_length: Option<usize>,
     /// Start content from this character index (default: 0),
     /// On return output starting at this character index, useful if a previous

crates/forge_app/src/tools/fetch.rs

@@ -23,6 +23,7 @@ uuid.workspace = true
 async-recursion.workspace = true
 tracing.workspace = true
 url.workspace = true
+merge.workspace = true
 
 [dev-dependencies]
 insta.workspace = true

crates/forge_domain/Cargo.toml

@@ -1,7 +1,9 @@
 use derive_more::derive::Display;
 use derive_setters::Setters;
+use merge::Merge;
 use serde::{Deserialize, Serialize};
 
+use crate::merge::Key;
 use crate::template::Template;
 use crate::{Environment, EventContext, ModelId, ToolName};
 
@@ -48,13 +50,17 @@ fn truth() -> bool {
     true
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Merge)]
 pub struct Agent {
     /// Flag to enable/disable tool support for this agent.
     #[serde(default)]
+    #[merge(strategy = crate::merge::bool::overwrite_false)]
     pub tool_supported: bool,
+    #[merge(strategy = crate::merge::std::overwrite)]
     pub id: AgentId,
-    pub model: ModelId,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub model: Option<ModelId>,
     pub description: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub system_prompt: Option<Template<SystemContext>>,
@@ -64,27 +70,33 @@ pub struct Agent {
     /// When set to true all user events will also contain a suggestions field
     /// that is prefilled with the matching information from vector store.
     #[serde(skip_serializing_if = "is_true", default)]
+    #[merge(strategy = crate::merge::bool::overwrite_false)]
     pub suggestions: bool,
 
     /// Suggests if the agent needs to maintain its state for the lifetime of
     /// the program.    
-    #[serde(skip_serializing_if = "is_true", default = "truth")]
+    #[serde(skip_serializing_if = "is_true", default)]
+    #[merge(strategy = crate::merge::bool::overwrite_false)]
     pub ephemeral: bool,
 
     /// Flag to enable/disable the agent. When disabled (false), the agent will
     /// be completely ignored during orchestration execution.
     #[serde(skip_serializing_if = "is_true", default = "truth")]
+    #[merge(strategy = crate::merge::bool::overwrite_false)]
     pub enable: bool,
 
     /// Tools that the agent can use    
-    #[serde(skip_serializing_if = "Vec::is_empty")]
+    #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    #[merge(strategy = crate::merge::vec::unify)]
     pub tools: Vec<ToolName>,
 
     #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    #[merge(strategy = crate::merge::vec::append)]
     pub transforms: Vec<Transform>,
 
     /// Used to specify the events the agent is interested in    
     #[serde(skip_serializing_if = "Vec::is_empty", default)]
+    #[merge(strategy = crate::merge::vec::unify)]
     pub subscribe: Vec<String>,
 
     /// Maximum number of turns the agent can take    
@@ -97,6 +109,14 @@ pub struct Agent {
     pub max_walker_depth: Option<usize>,
 }
 
+impl Key for Agent {
+    type Id = AgentId;
+
+    fn key(&self) -> &Self::Id {
+        &self.id
+    }
+}
+
 /// Transformations that can be applied to the agent's context before sending it
 /// upstream to the provider.
 #[derive(Debug, Clone, Serialize, Deserialize)]

crates/forge_domain/src/agent.rs

@@ -36,6 +36,9 @@ pub enum Error {
 
     #[error("Conversation not found: {0}")]
     ConversationNotFound(ConversationId),
+
+    #[error("Missing model for agent: {0}")]
+    MissingModel(AgentId),
 }
 
 pub type Result<A> = std::result::Result<A, Error>;

crates/forge_domain/src/error.rs

@@ -10,6 +10,7 @@ mod env;
 mod error;
 mod event;
 mod file;
+mod merge;
 mod message;
 mod model;
 mod orch;

crates/forge_domain/src/lib.rs

@@ -0,0 +1,39 @@
+pub mod std {
+    pub fn overwrite<T>(base: &mut T, other: T) {
+        *base = other;
+    }
+}
+pub mod vec {
+    pub use merge::vec::*;
+    use merge::Merge;
+
+    use super::Key;
+    pub fn unify<T: PartialEq>(base: &mut Vec<T>, other: Vec<T>) {
+        for other_item in other {
+            if !base.contains(&other_item) {
+                base.push(other_item);
+            }
+        }
+    }
+
+    pub fn unify_by_key<T: Merge + Key>(base: &mut Vec<T>, other: Vec<T>) {
+        for other_agent in other {
+            if let Some(base_agent) = base.iter_mut().find(|a| a.key() == other_agent.key()) {
+                // If the base contains an agent with the same Key, merge them
+                base_agent.merge(other_agent);
+            } else {
+                // Otherwise, append the other agent to the base list
+                base.push(other_agent);
+            }
+        }
+    }
+}
+
+pub mod bool {
+    pub use merge::bool::*;
+}
+
+pub trait Key {
+    type Id: Eq;
+    fn key(&self) -> &Self::Id;
+}

crates/forge_domain/src/merge.rs

@@ -338,7 +338,13 @@ impl<A: App> Orchestrator<A> {
             let response = self
                 .app
                 .provider_service()
-                .chat(&agent.model, context.clone())
+                .chat(
+                    agent
+                        .model
+                        .as_ref()
+                        .ok_or(Error::MissingModel(agent.id.clone()))?,
+                    context.clone(),
+                )
                 .await?;
             let ChatCompletionResult { tool_calls, content } =
                 self.collect_messages(&agent.id, response).await?;