diff --git a/Cargo.lock b/Cargo.lock index 92715eb9..b05c3ae4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3622,10 +3622,12 @@ version = "0.1.0" dependencies = [ "itertools 0.12.1", "nom", + "serde", "thiserror", "weaver_cache", "weaver_common", "weaver_diff", + "weaver_forge", "weaver_resolved_schema", "weaver_resolver", "weaver_semconv", diff --git a/crates/weaver_cache/src/lib.rs b/crates/weaver_cache/src/lib.rs index 0dbded3b..e0a7c35b 100644 --- a/crates/weaver_cache/src/lib.rs +++ b/crates/weaver_cache/src/lib.rs @@ -102,7 +102,11 @@ impl Cache { .expect("git_repo_dirs lock failed") .get(&repo_url) { - return Ok(git_repo_dir.path.clone()); + if let Some(subdir) = path { + return Ok(git_repo_dir.path.join(subdir)); + } else { + return Ok(git_repo_dir.path.clone()); + } } // Otherwise creates a tempdir for the repo and keeps track of it @@ -175,11 +179,11 @@ impl Cache { repo_url.clone(), GitRepo { temp_dir: git_repo_dir, - path: git_repo_path, + path: git_repo_path.clone(), }, ); - Ok(git_repo_pathbuf) + Ok(git_repo_path) } } diff --git a/crates/weaver_forge/src/error.rs b/crates/weaver_forge/src/error.rs index 14843bf3..51a0ccf0 100644 --- a/crates/weaver_forge/src/error.rs +++ b/crates/weaver_forge/src/error.rs @@ -3,7 +3,7 @@ //! Error types and utilities. use crate::error::Error::CompoundError; -use std::path::PathBuf; +use std::{path::PathBuf, str::FromStr}; use weaver_common::error::WeaverError; use weaver_resolved_schema::attribute::AttributeRef; @@ -139,6 +139,15 @@ impl WeaverError for Error { } } +#[must_use] +pub(crate) fn jinja_err_convert(e: minijinja::Error) -> Error { + Error::WriteGeneratedCodeFailed { + template: PathBuf::from_str(e.template_source().unwrap_or("")) + .expect("Template source should be path"), + error: format!("{}", e), + } +} + impl Error { /// Creates a compound error from a list of errors. /// Note: All compound errors are flattened. diff --git a/crates/weaver_forge/src/lib.rs b/crates/weaver_forge/src/lib.rs index f8e3bc3f..a92fd18d 100644 --- a/crates/weaver_forge/src/lib.rs +++ b/crates/weaver_forge/src/lib.rs @@ -156,6 +156,33 @@ impl TemplateEngine { }) } + /// Generate a template snippet from serializable context and a snippet identifier. + /// + /// # Arguments + /// + /// * `log` - The logger to use for logging + /// * `context` - The context to use when generating snippets. + /// * `snippet_id` - The template to use when rendering the snippet. + pub fn generate_snippet( + &self, + context: &T, + snippet_id: String, + ) -> Result { + // TODO - find the snippet by id. + + // Create a read-only context for the filter evaluations + let context = serde_json::to_value(context).map_err(|e| ContextSerializationFailed { + error: e.to_string(), + })?; + + let engine = self.template_engine()?; + let template = engine + .get_template(&snippet_id) + .map_err(error::jinja_err_convert)?; + let result = template.render(context).map_err(error::jinja_err_convert)?; + Ok(result) + } + /// Generate artifacts from a serializable context and a template directory, /// in parallel. /// diff --git a/crates/weaver_forge/src/registry.rs b/crates/weaver_forge/src/registry.rs index 3c2a0d62..1b851a12 100644 --- a/crates/weaver_forge/src/registry.rs +++ b/crates/weaver_forge/src/registry.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; use weaver_resolved_schema::attribute::Attribute; use weaver_resolved_schema::catalog::Catalog; use weaver_resolved_schema::lineage::GroupLineage; -use weaver_resolved_schema::registry::{Constraint, Registry}; +use weaver_resolved_schema::registry::{Constraint, Group, Registry}; use weaver_semconv::group::{GroupType, InstrumentSpec, SpanKindSpec}; use weaver_semconv::stability::Stability; @@ -102,6 +102,59 @@ pub struct TemplateGroup { pub lineage: Option, } +impl TemplateGroup { + /// Constructs a Template-friendly groups structure from resolved registry structures. + pub fn try_from_resolved(group: &Group, catalog: &Catalog) -> Result { + let mut errors = Vec::new(); + let id = group.id.clone(); + let group_type = group.r#type.clone(); + let brief = group.brief.clone(); + let note = group.note.clone(); + let prefix = group.prefix.clone(); + let extends = group.extends.clone(); + let stability = group.stability.clone(); + let deprecated = group.deprecated.clone(); + let constraints = group.constraints.clone(); + let attributes = group + .attributes + .iter() + .filter_map(|attr_ref| { + let attr = catalog.attribute(attr_ref).cloned(); + if attr.is_none() { + errors.push(Error::AttributeNotFound { + group_id: id.clone(), + attr_ref: *attr_ref, + }); + } + attr + }) + .collect(); + let lineage = group.lineage.clone(); + if !errors.is_empty() { + return Err(Error::CompoundError(errors)); + } + Ok(TemplateGroup { + id, + r#type: group_type, + brief, + note, + prefix, + extends, + stability, + deprecated, + constraints, + attributes, + span_kind: group.span_kind.clone(), + events: group.events.clone(), + metric_name: group.metric_name.clone(), + instrument: group.instrument.clone(), + unit: group.unit.clone(), + name: group.name.clone(), + lineage, + }) + } +} + impl TemplateRegistry { /// Create a new template registry from a resolved registry. pub fn try_from_resolved_registry( diff --git a/crates/weaver_semconv/src/group.rs b/crates/weaver_semconv/src/group.rs index f2dc7975..3dacc4e7 100644 --- a/crates/weaver_semconv/src/group.rs +++ b/crates/weaver_semconv/src/group.rs @@ -295,9 +295,7 @@ impl Display for InstrumentSpec { #[cfg(test)] mod tests { - use crate::attribute::{AttributeSpec, AttributeType, Examples, PrimitiveOrArrayTypeSpec}; - use crate::group::{GroupSpec, GroupType, SpanKindSpec}; - use crate::stability::Stability; + use crate::attribute::Examples; use crate::Error::{CompoundError, InvalidAttribute, InvalidGroup, InvalidMetric}; use super::*; diff --git a/crates/weaver_semconv_gen/Cargo.toml b/crates/weaver_semconv_gen/Cargo.toml index 818a9fd9..34e5c29e 100644 --- a/crates/weaver_semconv_gen/Cargo.toml +++ b/crates/weaver_semconv_gen/Cargo.toml @@ -12,6 +12,7 @@ rust-version.workspace = true weaver_common = { path = "../weaver_common" } weaver_cache = { path = "../weaver_cache" } weaver_diff = { path = "../weaver_diff" } +weaver_forge = { path = "../weaver_forge" } weaver_resolver = { path = "../weaver_resolver" } weaver_resolved_schema = { path = "../weaver_resolved_schema" } weaver_semconv = { path = "../weaver_semconv" } @@ -19,6 +20,7 @@ itertools = "0.12.1" nom = "7.1.3" thiserror.workspace = true +serde.workspace = true [lints] workspace = true diff --git a/crates/weaver_semconv_gen/README.md b/crates/weaver_semconv_gen/README.md index 60f4a2e6..39e68432 100644 --- a/crates/weaver_semconv_gen/README.md +++ b/crates/weaver_semconv_gen/README.md @@ -2,4 +2,46 @@ Status: **Work-In-Progress** -This crate duplicates the semconv templating from open-telemetry/build-tools. \ No newline at end of file +This crate duplicates the semconv templating from open-telemetry/build-tools. It enables +generating "snippet" templates inside existing Markdown documents. + + +## Snippet Definitions + +This crate can update (or diff) (`.md`) files with snippets, like so: + +```markdown +# My Markdown file + + +This content will be replaced by generated snippet. + +``` + +Snippets can be defined with the following pseudo-grammar: + +``` +SNIPPET_TAG = "semconv" GROUP_ID SNIPPET_ARGS? +GROUP_ID = ('A'-'Z', 'a'-'z', '.', '_', '-')+ +SNIPPET_ARGS = "(" SNIPPET_ARG ("," SNIPPET_ARG)* ")" +SNIPPET_ARG = + "full" | + "metric_table" | + "omit_requirement_level" | + ("tag" "=" ('A'-'Z','a'-'z','0'-'9')+) +``` + + +## Snippet Templates + +You can use `weaver_forge` and `minijinja` templates for snippet generation. When doing so, a template named +`snippet.md.j2` will be used for all snippet generation. + +The template will be passed the following context variables: + +- `group`: The resolved semantic convention group, referenced by id in the snippet tag. +- `snippet_type`: Either `metric_table` or `attribute_table`, based on arguments to the snippet tag. +- `tag_filter`: The set of all values defined as tag filters. +- `attribute_registry_base_url`: Base url to use when making attribute registry links. + +Otherwise, the template will be given all filters, tests and functions defined in `weaver_forge`. \ No newline at end of file diff --git a/crates/weaver_semconv_gen/allowed-external-types.toml b/crates/weaver_semconv_gen/allowed-external-types.toml index f66bcc2b..0011217d 100644 --- a/crates/weaver_semconv_gen/allowed-external-types.toml +++ b/crates/weaver_semconv_gen/allowed-external-types.toml @@ -7,5 +7,8 @@ allowed_external_types = [ "weaver_semconv::path::RegistryPath", "weaver_resolver::Error", "weaver_cache::Cache", - "weaver_common::error::WeaverError" + "weaver_cache::Error", + "weaver_common::error::WeaverError", + "weaver_forge::error::Error", + "weaver_forge::TemplateEngine" ] diff --git a/crates/weaver_semconv_gen/data/templates.md b/crates/weaver_semconv_gen/data/templates.md new file mode 100644 index 00000000..771157cf --- /dev/null +++ b/crates/weaver_semconv_gen/data/templates.md @@ -0,0 +1,7 @@ + +attribute_table: registry.user_agent + + + +metric_table: metric.http.server.request.duration + diff --git a/crates/weaver_semconv_gen/src/lib.rs b/crates/weaver_semconv_gen/src/lib.rs index e34028ab..8cd3c266 100644 --- a/crates/weaver_semconv_gen/src/lib.rs +++ b/crates/weaver_semconv_gen/src/lib.rs @@ -6,10 +6,14 @@ use std::fs; +use serde::Serialize; use weaver_cache::Cache; use weaver_common::error::{format_errors, WeaverError}; use weaver_diff::diff_output; +use weaver_forge::registry::TemplateGroup; +use weaver_forge::TemplateEngine; use weaver_resolved_schema::attribute::{Attribute, AttributeRef}; +use weaver_resolved_schema::catalog::Catalog; use weaver_resolved_schema::registry::{Group, Registry}; use weaver_resolved_schema::ResolvedTelemetrySchema; use weaver_resolver::SchemaResolver; @@ -77,6 +81,14 @@ pub enum Error { #[error(transparent)] ResolverError(#[from] weaver_resolver::Error), + /// Errors from using weaver_cache. + #[error(transparent)] + CacheError(#[from] weaver_cache::Error), + + /// Errors from using weaver_forge. + #[error(transparent)] + ForgeError(#[from] weaver_forge::error::Error), + /// A container for multiple errors. #[error("{:?}", format_errors(.0))] CompoundError(Vec), @@ -153,30 +165,41 @@ impl GenerateMarkdownArgs { _ => None, }) } -} -/// Constructs a markdown snippet (without header/closer) -fn generate_markdown_snippet( - lookup: &ResolvedSemconvRegistry, - args: GenerateMarkdownArgs, - attribute_registry_base_url: Option<&str>, -) -> Result { - let mut ctx = GenerateMarkdownContext::default(); - let mut result = String::new(); - if args.is_metric_table() { - let view = MetricView::try_new(args.id.as_str(), lookup)?; - view.generate_markdown(&mut result, &mut ctx)?; - } else { - let other = AttributeTableView::try_new(args.id.as_str(), lookup)?; - other.generate_markdown(&mut result, &args, &mut ctx, attribute_registry_base_url)?; + /// Returns all tag filters in a list. + fn tag_filters(&self) -> Vec<&str> { + self.args + .iter() + .find_map(|arg| match arg { + MarkdownGenParameters::Tag(value) => Some(value.as_str()), + _ => None, + }) + .into_iter() + .collect() } - Ok(result) +} + +/// This struct is passed into markdown snippets for generation. +#[derive(Serialize)] +struct MarkdownSnippetContext { + group: TemplateGroup, + snippet_type: SnippetType, + tag_filter: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + attribute_registry_base_url: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +enum SnippetType { + AttributeTable, + MetricTable, } // TODO - This entire function could be optimised and reworked. fn update_markdown_contents( contents: &str, - lookup: &ResolvedSemconvRegistry, + generator: &SnippetGenerator, attribute_registry_base_url: Option<&str>, ) -> Result { let mut result = String::new(); @@ -199,7 +222,8 @@ fn update_markdown_contents( if parser::is_markdown_snippet_directive(line) { handling_snippet = true; let arg = parser::parse_markdown_snippet_directive(line)?; - let snippet = generate_markdown_snippet(lookup, arg, attribute_registry_base_url)?; + let snippet = + generator.generate_markdown_snippet(arg, attribute_registry_base_url)?; result.push_str(&snippet); } } @@ -210,13 +234,13 @@ fn update_markdown_contents( /// Updates a single markdown file using the resolved schema. pub fn update_markdown( file: &str, - lookup: &ResolvedSemconvRegistry, + generator: &SnippetGenerator, dry_run: bool, attribute_registry_base_url: Option<&str>, ) -> Result<(), Error> { let original_markdown = fs::read_to_string(file)?.replace("\r\n", "\n"); let updated_markdown = - update_markdown_contents(&original_markdown, lookup, attribute_registry_base_url)?; + update_markdown_contents(&original_markdown, generator, attribute_registry_base_url)?; if !dry_run { fs::write(file, updated_markdown)?; Ok(()) @@ -230,27 +254,111 @@ pub fn update_markdown( } } +/// State we need to generate markdown snippets from configuration. +pub struct SnippetGenerator { + lookup: ResolvedSemconvRegistry, + template_engine: Option, +} + +impl SnippetGenerator { + // TODO - move registry base url into state of the struct... + fn generate_markdown_snippet( + &self, + args: GenerateMarkdownArgs, + attribute_registry_base_url: Option<&str>, + ) -> Result { + if let Some(template) = &self.template_engine { + // TODO - define context. + let snippet_type = if args.is_metric_table() { + SnippetType::MetricTable + } else { + SnippetType::AttributeTable + }; + let group = self + .lookup + .find_group(&args.id) + .ok_or(Error::GroupNotFound { + id: args.id.clone(), + }) + .and_then(|g| Ok(TemplateGroup::try_from_resolved(g, self.lookup.catalog())?))?; + // Context is the JSON sent to the jinja template engine. + let context = MarkdownSnippetContext { + group: group.clone(), + snippet_type, + tag_filter: args + .tag_filters() + .into_iter() + .map(|s| s.to_owned()) + .collect(), + attribute_registry_base_url: attribute_registry_base_url.map(|s| s.to_owned()), + }; + // We automatically default to specific file for the snippet types. + let snippet_template_file = "snippet.md.j2"; + let mut result = + template.generate_snippet(&context, snippet_template_file.to_owned())?; + result.push('\n'); + Ok(result) + } else { + self.generate_legacy_markdown_snippet(args, attribute_registry_base_url) + } + } + + fn generate_legacy_markdown_snippet( + &self, + args: GenerateMarkdownArgs, + attribute_registry_base_url: Option<&str>, + ) -> Result { + let mut ctx = GenerateMarkdownContext::default(); + let mut result = String::new(); + if args.is_metric_table() { + let view = MetricView::try_new(args.id.as_str(), &self.lookup)?; + view.generate_markdown(&mut result, &mut ctx)?; + } else { + let other = AttributeTableView::try_new(args.id.as_str(), &self.lookup)?; + other.generate_markdown(&mut result, &args, &mut ctx, attribute_registry_base_url)?; + } + Ok(result) + } + + /// Resolve semconv registry (possibly from git), and make it available for rendering. + pub fn try_from_url( + registry_path: RegistryPath, + cache: &Cache, + template_engine: Option, + ) -> Result { + let registry = ResolvedSemconvRegistry::try_from_url(registry_path, cache)?; + Ok(SnippetGenerator { + lookup: registry, + template_engine, + }) + } + + // Used in tests + #[allow(dead_code)] + fn try_from_path( + path_pattern: &str, + template_engine: Option, + ) -> Result { + let cache = Cache::try_new()?; + Self::try_from_url( + RegistryPath::Local { + path_pattern: path_pattern.to_owned(), + }, + &cache, + template_engine, + ) + } +} + /// The resolved Semantic Convention repository that is used to drive snipper generation. -pub struct ResolvedSemconvRegistry { +struct ResolvedSemconvRegistry { schema: ResolvedTelemetrySchema, registry_id: String, } impl ResolvedSemconvRegistry { - /// Resolve the semantic convention registry and make it available for rendering markdown snippets. - pub fn try_from_path(path_pattern: &str) -> Result { - let registry_id = "semantic_conventions"; - let mut registry = SemConvRegistry::try_from_path_pattern(registry_id, path_pattern)?; - let schema = SchemaResolver::resolve_semantic_convention_registry(&mut registry)?; - let lookup = ResolvedSemconvRegistry { - schema, - registry_id: registry_id.into(), - }; - Ok(lookup) - } - /// Resolve semconv registry (possibly from git), and make it available for rendering. - pub fn try_from_url( + fn try_from_url( registry_path: RegistryPath, cache: &Cache, ) -> Result { @@ -269,6 +377,10 @@ impl ResolvedSemconvRegistry { self.schema.registry(self.registry_id.as_str()) } + fn catalog(&self) -> &Catalog { + &self.schema.catalog + } + fn find_group(&self, id: &str) -> Option<&Group> { self.my_registry() .and_then(|r| r.groups.iter().find(|g| g.id == id)) @@ -285,7 +397,9 @@ mod tests { use std::fs; use std::path::PathBuf; - use crate::{update_markdown, Error, ResolvedSemconvRegistry}; + use weaver_forge::{GeneratorConfig, TemplateEngine}; + + use crate::{update_markdown, Error, SnippetGenerator}; fn force_print_error(result: Result) -> T { match result { @@ -294,9 +408,26 @@ mod tests { } } + #[test] + fn test_template_engine() -> Result<(), Error> { + let template = TemplateEngine::try_new("markdown", GeneratorConfig::default())?; + let generator = SnippetGenerator::try_from_path("data", Some(template))?; + let attribute_registry_url = "/docs/attributes-registry"; + // Now we should check a snippet. + let test = "data/templates.md"; + println!("--- Running template engine test: {test} ---"); + force_print_error(update_markdown( + test, + &generator, + true, + Some(attribute_registry_url), + )); + Ok(()) + } + #[test] fn test_http_semconv() -> Result<(), Error> { - let lookup = ResolvedSemconvRegistry::try_from_path("data/**/*.yaml")?; + let lookup = SnippetGenerator::try_from_path("data", None)?; let attribute_registry_url = "/docs/attributes-registry"; // Check our test files. for test in [ @@ -331,8 +462,8 @@ mod tests { } fn run_legacy_test(path: PathBuf) -> Result<(), Error> { - let semconv_path = format!("{}/*.yaml", path.display()); - let lookup = ResolvedSemconvRegistry::try_from_path(&semconv_path)?; + let semconv_path = format!("{}", path.display()); + let lookup = SnippetGenerator::try_from_path(&semconv_path, None)?; let test_path = path.join("test.md").display().to_string(); // Attempts to update the test - will fail if there is any difference in the generated markdown. update_markdown(&test_path, &lookup, true, None) diff --git a/crates/weaver_semconv_gen/templates/markdown/snippet.md.j2 b/crates/weaver_semconv_gen/templates/markdown/snippet.md.j2 new file mode 100644 index 00000000..fcdcf110 --- /dev/null +++ b/crates/weaver_semconv_gen/templates/markdown/snippet.md.j2 @@ -0,0 +1 @@ +{{ snippet_type }}: {{ group.id }} diff --git a/src/registry/update_markdown.rs b/src/registry/update_markdown.rs index 22251df4..9a9248f9 100644 --- a/src/registry/update_markdown.rs +++ b/src/registry/update_markdown.rs @@ -8,7 +8,8 @@ use clap::Args; use weaver_cache::Cache; use weaver_common::error::ExitIfError; use weaver_common::Logger; -use weaver_semconv_gen::{update_markdown, ResolvedSemconvRegistry}; +use weaver_forge::{GeneratorConfig, TemplateEngine}; +use weaver_semconv_gen::{update_markdown, SnippetGenerator}; /// Parameters for the `registry update-markdown` sub-command #[derive(Debug, Args)] @@ -37,6 +38,19 @@ pub struct RegistryUpdateMarkdownArgs { /// If provided, all attributes will be linked here. #[arg(long)] pub attribute_registry_base_url: Option, + + /// Path to the directory where the templates are located. + /// Default is the `templates` directory. + /// Note: `registry update-markdown` will look for a specific jinja template: + /// {templates}/{target}/snippet.md.j2. + #[arg(short = 't', long, default_value = "templates")] + pub templates: String, + + /// If provided, the target to generate snippets with. + /// Note: `registry update-markdown` will look for a specific jinja template: + /// {templates}/{target}/snippet.md.j2. + #[arg(long)] + pub target: Option, } /// Update markdown files. @@ -50,10 +64,16 @@ pub(crate) fn command( let extension = path.extension().unwrap_or_else(|| std::ffi::OsStr::new("")); path.is_file() && extension == "md" } + // Construct a generator if we were given a `--target` argument. + let generator = args.target.as_ref().map(|target| { + TemplateEngine::try_new(&format!("registry/{}", target), GeneratorConfig::default()) + .exit_if_error(log.clone()) + }); - let registry = ResolvedSemconvRegistry::try_from_url( + let generator = SnippetGenerator::try_from_url( semconv_registry_path_from(&args.registry, &args.registry_git_sub_dir), cache, + generator, ) .exit_if_error(log.clone()); log.success("Registry resolved successfully"); @@ -73,7 +93,7 @@ pub(crate) fn command( log.info(&format!("{}: ${}", operation, entry.path().display())); if let Err(error) = update_markdown( &entry.path().display().to_string(), - ®istry, + &generator, args.dry_run, args.attribute_registry_base_url.as_deref(), ) {