From ee9383b50f692876217bcebefc1c53735c6abc95 Mon Sep 17 00:00:00 2001 From: laststylebender Date: Mon, 20 Jan 2025 12:56:55 +0530 Subject: [PATCH 1/5] - add supported models in e2e --- crates/forge_app/src/service/api.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/forge_app/src/service/api.rs b/crates/forge_app/src/service/api.rs index a9362d542..58eafc7e1 100644 --- a/crates/forge_app/src/service/api.rs +++ b/crates/forge_app/src/service/api.rs @@ -154,6 +154,15 @@ mod tests { async fn test_e2e() { const MAX_RETRIES: usize = 3; const MATCH_THRESHOLD: f64 = 0.7; // 70% of crates must be found + const SUPPORTED_MODELS: &[&str] = &[ + "anthropic/claude-3.5-sonnet:beta", + "openai/gpt-4o-2024-11-20", + "anthropic/claude-3.5-sonnet", + "openai/gpt-4o", + "openai/gpt-4o-mini", + "google/gemini-flash-1.5", + "anthropic/claude-3-sonnet", + ]; let api = Live::new().await.unwrap(); let task = include_str!("./api_task.md"); From 2cc232fa778f4984f17e9a832cc8a6b0d75613af Mon Sep 17 00:00:00 2001 From: laststylebender Date: Mon, 20 Jan 2025 13:15:48 +0530 Subject: [PATCH 2/5] - make e2e test parallel --- Cargo.lock | 1 + crates/forge_app/Cargo.toml | 1 + crates/forge_app/src/service/api.rs | 146 ++++++++++++++++------------ 3 files changed, 85 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 953bcbc46..b1cf514a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -886,6 +886,7 @@ dependencies = [ "forge_prompt", "forge_tool", "forge_walker", + "futures", "handlebars", "insta", "moka2", diff --git a/crates/forge_app/Cargo.toml b/crates/forge_app/Cargo.toml index 28e1bd682..92f542656 100644 --- a/crates/forge_app/Cargo.toml +++ b/crates/forge_app/Cargo.toml @@ -35,6 +35,7 @@ schemars = "0.8.21" anyhow = "1.0.75" [dev-dependencies] +futures = "0.3.31" insta = "1.41.1" pretty_assertions = "1.4.1" tempfile = "3.10.1" diff --git a/crates/forge_app/src/service/api.rs b/crates/forge_app/src/service/api.rs index 58eafc7e1..1b9279276 100644 --- a/crates/forge_app/src/service/api.rs +++ b/crates/forge_app/src/service/api.rs @@ -147,11 +147,15 @@ impl APIService for Live { mod tests { use forge_domain::ModelId; use tokio_stream::StreamExt; + use futures::future::join_all; use super::*; #[tokio::test] async fn test_e2e() { + let api = Live::new().await.unwrap(); + let task = include_str!("./api_task.md"); + const MAX_RETRIES: usize = 3; const MATCH_THRESHOLD: f64 = 0.7; // 70% of crates must be found const SUPPORTED_MODELS: &[&str] = &[ @@ -164,71 +168,87 @@ mod tests { "anthropic/claude-3-sonnet", ]; - let api = Live::new().await.unwrap(); - let task = include_str!("./api_task.md"); - let request = ChatRequest::new(ModelId::new("anthropic/claude-3.5-sonnet"), task); - - let expected_crates = [ - "forge_app", - "forge_ci", - "forge_domain", - "forge_main", - "forge_open_router", - "forge_prompt", - "forge_tool", - "forge_tool_macros", - "forge_walker", - ]; - - let mut last_error = None; - - for attempt in 0..MAX_RETRIES { - let response = api - .chat(request.clone()) - .await - .unwrap() - .filter_map(|message| match message.unwrap() { - ChatResponse::Text(text) => Some(text), - _ => None, - }) - .collect::>() - .await - .join("") - .trim() - .to_string(); - - let found_crates: Vec<&str> = expected_crates - .iter() - .filter(|&crate_name| response.contains(&format!("{}", crate_name))) - .cloned() - .collect(); - - let match_percentage = found_crates.len() as f64 / expected_crates.len() as f64; - - if match_percentage >= MATCH_THRESHOLD { - println!( - "Successfully found {:.2}% of expected crates", - match_percentage * 100.0 - ); - return; + let test_futures = SUPPORTED_MODELS.iter().map(|&model| { + let api = api.clone(); + let task = task.to_string(); + + async move { + let request = ChatRequest::new(ModelId::new(model), task); + let expected_crates = [ + "forge_app", + "forge_ci", + "forge_domain", + "forge_main", + "forge_open_router", + "forge_prompt", + "forge_tool", + "forge_tool_macros", + "forge_walker", + ]; + + for attempt in 0..MAX_RETRIES { + let response = api + .chat(request.clone()) + .await + .unwrap() + .filter_map(|message| match message.unwrap() { + ChatResponse::Text(text) => Some(text), + _ => None, + }) + .collect::>() + .await + .join("") + .trim() + .to_string(); + + let found_crates: Vec<&str> = expected_crates + .iter() + .filter(|&crate_name| response.contains(&format!("{}", crate_name))) + .cloned() + .collect(); + + let match_percentage = found_crates.len() as f64 / expected_crates.len() as f64; + + if match_percentage >= MATCH_THRESHOLD { + println!( + "[{}] Successfully found {:.2}% of expected crates", + model, + match_percentage * 100.0 + ); + return Ok::<_, String>(()); + } + + if attempt < MAX_RETRIES - 1 { + println!( + "[{}] Attempt {}/{}: Found {}/{} crates: {:?}", + model, + attempt + 1, + MAX_RETRIES, + found_crates.len(), + expected_crates.len(), + found_crates + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } else { + return Err(format!( + "[{}] Failed: Found only {}/{} crates: {:?}", + model, + found_crates.len(), + expected_crates.len(), + found_crates + )); + } + } + + unreachable!() } + }); - last_error = Some(format!( - "Attempt {}: Only found {}/{} crates: {:?}", - attempt + 1, - found_crates.len(), - expected_crates.len(), - found_crates - )); - - // Add a small delay between retries to allow for different LLM generations - tokio::time::sleep(std::time::Duration::from_secs(2)).await; + let results = join_all(test_futures).await; + let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect(); + + if !errors.is_empty() { + panic!("Test failures:\n{}", errors.join("\n")); } - - panic!( - "Failed after {} attempts. Last error: {}", - MAX_RETRIES, - last_error.unwrap_or_default() - ); } } From e6ae627dee3d1840df228658520252c0b5df854b Mon Sep 17 00:00:00 2001 From: laststylebender Date: Mon, 20 Jan 2025 13:41:53 +0530 Subject: [PATCH 3/5] - let LLM, figure out the tool func call structure. --- crates/forge_app/src/prompts/title.md | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/forge_app/src/prompts/title.md b/crates/forge_app/src/prompts/title.md index 785ea2947..db8c0d64e 100644 --- a/crates/forge_app/src/prompts/title.md +++ b/crates/forge_app/src/prompts/title.md @@ -48,7 +48,6 @@ Example output structure: 8. Potential objections: [List any weaknesses for each title] 9. SEO alignment: [Reflect on SEO best practices for each title] 10. Selected title: [Explain your choice and its alignment] -11. Tool call preparation: generate_title(title: "Selected Title") From 903bc388730ec875e014696b6954b401aed77f0a Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 14:38:09 +0000 Subject: [PATCH 4/5] [autofix.ci] apply automated fixes --- crates/forge_app/src/service/api.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/forge_app/src/service/api.rs b/crates/forge_app/src/service/api.rs index 1b9279276..dcec554f5 100644 --- a/crates/forge_app/src/service/api.rs +++ b/crates/forge_app/src/service/api.rs @@ -146,8 +146,8 @@ impl APIService for Live { #[cfg(test)] mod tests { use forge_domain::ModelId; - use tokio_stream::StreamExt; use futures::future::join_all; + use tokio_stream::StreamExt; use super::*; @@ -171,7 +171,7 @@ mod tests { let test_futures = SUPPORTED_MODELS.iter().map(|&model| { let api = api.clone(); let task = task.to_string(); - + async move { let request = ChatRequest::new(ModelId::new(model), task); let expected_crates = [ @@ -203,7 +203,9 @@ mod tests { let found_crates: Vec<&str> = expected_crates .iter() - .filter(|&crate_name| response.contains(&format!("{}", crate_name))) + .filter(|&crate_name| { + response.contains(&format!("{}", crate_name)) + }) .cloned() .collect(); @@ -239,14 +241,14 @@ mod tests { )); } } - + unreachable!() } }); let results = join_all(test_futures).await; let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect(); - + if !errors.is_empty() { panic!("Test failures:\n{}", errors.join("\n")); } From ef586d3c3f9b3aa9fc3067d6ab4b80fdef857ff2 Mon Sep 17 00:00:00 2001 From: laststylebender Date: Mon, 20 Jan 2025 22:41:09 +0530 Subject: [PATCH 5/5] refactor(env): simplify working directory handling in environment service --- crates/forge_app/src/service/api.rs | 11 +++++++---- crates/forge_app/src/service/env.rs | 27 ++++++++++++++++++--------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/crates/forge_app/src/service/api.rs b/crates/forge_app/src/service/api.rs index dcec554f5..cd5fad49e 100644 --- a/crates/forge_app/src/service/api.rs +++ b/crates/forge_app/src/service/api.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use std::sync::Arc; use anyhow::Result; @@ -29,7 +30,7 @@ pub trait APIService: Send + Sync { impl Service { pub async fn api_service() -> Result { - Live::new().await + Live::new(std::env::current_dir()?).await } } @@ -45,8 +46,8 @@ struct Live { } impl Live { - async fn new() -> Result { - let env = Service::environment_service().get().await?; + async fn new(cwd: PathBuf) -> Result { + let env = Service::environment_service(cwd).get().await?; let cwd: String = env.cwd.clone(); let provider = Arc::new(Service::provider_service(env.api_key.clone())); @@ -145,6 +146,8 @@ impl APIService for Live { #[cfg(test)] mod tests { + use std::path::Path; + use forge_domain::ModelId; use futures::future::join_all; use tokio_stream::StreamExt; @@ -153,7 +156,7 @@ mod tests { #[tokio::test] async fn test_e2e() { - let api = Live::new().await.unwrap(); + let api = Live::new(Path::new("../../").to_path_buf()).await.unwrap(); let task = include_str!("./api_task.md"); const MAX_RETRIES: usize = 3; diff --git a/crates/forge_app/src/service/env.rs b/crates/forge_app/src/service/env.rs index 58dc10d76..799817b50 100644 --- a/crates/forge_app/src/service/env.rs +++ b/crates/forge_app/src/service/env.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use anyhow::Result; use forge_domain::Environment; use forge_walker::Walker; @@ -11,19 +13,22 @@ pub trait EnvironmentService { } impl Service { - pub fn environment_service() -> impl EnvironmentService { - Live::new() + pub fn environment_service(base_dir: PathBuf) -> impl EnvironmentService { + Live::new(base_dir) } } -struct Live(Mutex>); +struct Live { + env: Mutex>, + base_dir: PathBuf, +} impl Live { - pub fn new() -> Self { - Self(Mutex::new(None)) + pub fn new(base_dir: PathBuf) -> Self { + Self { env: Mutex::new(None), base_dir } } - async fn from_env() -> Result { + async fn from_env(cwd: Option) -> Result { dotenv::dotenv().ok(); let api_key = std::env::var("FORGE_KEY").expect("FORGE_KEY must be set"); let large_model_id = @@ -31,7 +36,11 @@ impl Live { let small_model_id = std::env::var("FORGE_SMALL_MODEL").unwrap_or("anthropic/claude-3.5-haiku".to_owned()); - let cwd = std::env::current_dir()?; + let cwd = if let Some(cwd) = cwd { + cwd + } else { + std::env::current_dir()? + }; let files = match Walker::new(cwd.clone()) .with_max_depth(usize::MAX) .get() @@ -65,12 +74,12 @@ impl Live { #[async_trait::async_trait] impl EnvironmentService for Live { async fn get(&self) -> Result { - let mut guard = self.0.lock().await; + let mut guard = self.env.lock().await; if let Some(env) = guard.as_ref() { return Ok(env.clone()); } else { - *guard = Some(Live::from_env().await?); + *guard = Some(Live::from_env(Some(self.base_dir.clone())).await?); Ok(guard.as_ref().unwrap().clone()) } }