Generate markdown snippets using weaver_forge templates (#141)

* Prototype snippets on templates

Create a prototype where markdown snippets are generated using Jinja / weaver_forge.

- Fix an issue in cache/git resolver now that semconv repo has non-model yaml files.
- Add new hooks in weaver_forge for generating just strings from templates.
- Add optional weaver_forge support to weaver_semconv_gen for snippet generation.

Lots of things that should be config are still hardcoded.

* Add docs for snippet feature

- Update to only use ONE snippet template for all snippets.
- Update context for snippets to include attribute registry base url
- Update documentation.

* Fix clippy

* Update allowed external types for semconv_gen

* Remove publicly exposed jinja error.

* Add weaver cache error to allowed external types.

* Fix rustfmt issue.

* Fix gen to use template-group instead of raw group.

* Fix clippy issue

* Fix from review.

crates/weaver_cache/src/lib.rs

@@ -102,7 +102,11 @@ impl Cache {
             .expect("git_repo_dirs lock failed")
             .get(&repo_url)
         {
-            return Ok(git_repo_dir.path.clone());
+            if let Some(subdir) = path {
+                return Ok(git_repo_dir.path.join(subdir));
+            } else {
+                return Ok(git_repo_dir.path.clone());
+            }
         }
 
         // Otherwise creates a tempdir for the repo and keeps track of it
@@ -175,11 +179,11 @@ impl Cache {
                 repo_url.clone(),
                 GitRepo {
                     temp_dir: git_repo_dir,
-                    path: git_repo_path,
+                    path: git_repo_path.clone(),
                 },
             );
 
-        Ok(git_repo_pathbuf)
+        Ok(git_repo_path)
     }
 }
 

crates/weaver_forge/src/error.rs

@@ -3,7 +3,7 @@
 //! Error types and utilities.
 
 use crate::error::Error::CompoundError;
-use std::path::PathBuf;
+use std::{path::PathBuf, str::FromStr};
 use weaver_common::error::WeaverError;
 use weaver_resolved_schema::attribute::AttributeRef;
 
@@ -139,6 +139,15 @@ impl WeaverError<Error> for Error {
     }
 }
 
+#[must_use]
+pub(crate) fn jinja_err_convert(e: minijinja::Error) -> Error {
+    Error::WriteGeneratedCodeFailed {
+        template: PathBuf::from_str(e.template_source().unwrap_or(""))
+            .expect("Template source should be path"),
+        error: format!("{}", e),
+    }
+}
+
 impl Error {
     /// Creates a compound error from a list of errors.
     /// Note: All compound errors are flattened.

crates/weaver_forge/src/lib.rs

@@ -156,6 +156,33 @@ impl TemplateEngine {
         })
     }
 
+    /// Generate a template snippet from serializable context and a snippet identifier.
+    ///
+    /// # Arguments
+    ///
+    /// * `log` - The logger to use for logging
+    /// * `context` - The context to use when generating snippets.
+    /// * `snippet_id` - The template to use when rendering the snippet.
+    pub fn generate_snippet<T: Serialize>(
+        &self,
+        context: &T,
+        snippet_id: String,
+    ) -> Result<String, Error> {
+        // TODO - find the snippet by id.
+
+        // Create a read-only context for the filter evaluations
+        let context = serde_json::to_value(context).map_err(|e| ContextSerializationFailed {
+            error: e.to_string(),
+        })?;
+
+        let engine = self.template_engine()?;
+        let template = engine
+            .get_template(&snippet_id)
+            .map_err(error::jinja_err_convert)?;
+        let result = template.render(context).map_err(error::jinja_err_convert)?;
+        Ok(result)
+    }
+
     /// Generate artifacts from a serializable context and a template directory,
     /// in parallel.
     ///

crates/weaver_forge/src/registry.rs

@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize};
 use weaver_resolved_schema::attribute::Attribute;
 use weaver_resolved_schema::catalog::Catalog;
 use weaver_resolved_schema::lineage::GroupLineage;
-use weaver_resolved_schema::registry::{Constraint, Registry};
+use weaver_resolved_schema::registry::{Constraint, Group, Registry};
 use weaver_semconv::group::{GroupType, InstrumentSpec, SpanKindSpec};
 use weaver_semconv::stability::Stability;
 
@@ -102,6 +102,59 @@ pub struct TemplateGroup {
     pub lineage: Option<GroupLineage>,
 }
 
+impl TemplateGroup {
+    /// Constructs a Template-friendly groups structure from resolved registry structures.
+    pub fn try_from_resolved(group: &Group, catalog: &Catalog) -> Result<Self, Error> {
+        let mut errors = Vec::new();
+        let id = group.id.clone();
+        let group_type = group.r#type.clone();
+        let brief = group.brief.clone();
+        let note = group.note.clone();
+        let prefix = group.prefix.clone();
+        let extends = group.extends.clone();
+        let stability = group.stability.clone();
+        let deprecated = group.deprecated.clone();
+        let constraints = group.constraints.clone();
+        let attributes = group
+            .attributes
+            .iter()
+            .filter_map(|attr_ref| {
+                let attr = catalog.attribute(attr_ref).cloned();
+                if attr.is_none() {
+                    errors.push(Error::AttributeNotFound {
+                        group_id: id.clone(),
+                        attr_ref: *attr_ref,
+                    });
+                }
+                attr
+            })
+            .collect();
+        let lineage = group.lineage.clone();
+        if !errors.is_empty() {
+            return Err(Error::CompoundError(errors));
+        }
+        Ok(TemplateGroup {
+            id,
+            r#type: group_type,
+            brief,
+            note,
+            prefix,
+            extends,
+            stability,
+            deprecated,
+            constraints,
+            attributes,
+            span_kind: group.span_kind.clone(),
+            events: group.events.clone(),
+            metric_name: group.metric_name.clone(),
+            instrument: group.instrument.clone(),
+            unit: group.unit.clone(),
+            name: group.name.clone(),
+            lineage,
+        })
+    }
+}
+
 impl TemplateRegistry {
     /// Create a new template registry from a resolved registry.
     pub fn try_from_resolved_registry(

crates/weaver_semconv/src/group.rs

@@ -295,9 +295,7 @@ impl Display for InstrumentSpec {
 
 #[cfg(test)]
 mod tests {
-    use crate::attribute::{AttributeSpec, AttributeType, Examples, PrimitiveOrArrayTypeSpec};
-    use crate::group::{GroupSpec, GroupType, SpanKindSpec};
-    use crate::stability::Stability;
+    use crate::attribute::Examples;
     use crate::Error::{CompoundError, InvalidAttribute, InvalidGroup, InvalidMetric};
 
     use super::*;

crates/weaver_semconv_gen/Cargo.toml

@@ -12,13 +12,15 @@ rust-version.workspace = true
 weaver_common = { path = "../weaver_common" }
 weaver_cache = { path = "../weaver_cache" }
 weaver_diff = { path = "../weaver_diff" }
+weaver_forge = { path = "../weaver_forge" }
 weaver_resolver = { path = "../weaver_resolver" }
 weaver_resolved_schema = { path = "../weaver_resolved_schema" }
 weaver_semconv = { path = "../weaver_semconv" }
 itertools = "0.12.1"
 nom = "7.1.3"
 
 thiserror.workspace = true
+serde.workspace = true
 
 [lints]
 workspace = true

crates/weaver_semconv_gen/README.md

@@ -2,4 +2,46 @@
 
 Status: **Work-In-Progress**
 
-This crate duplicates the semconv templating from open-telemetry/build-tools.
+This crate duplicates the semconv templating from open-telemetry/build-tools.  It enables
+generating "snippet" templates inside existing Markdown documents.
+
+
+## Snippet Definitions
+
+This crate can update (or diff) (`.md`) files with snippets, like so:
+
+```markdown
+# My Markdown file
+
+<!-- semconv some.group.id -->
+This content will be replaced by generated snippet.
+<!-- endsemconv -->
+```
+
+Snippets can be defined with the following pseudo-grammar:
+
+```
+SNIPPET_TAG = "semconv" GROUP_ID SNIPPET_ARGS?
+GROUP_ID = ('A'-'Z', 'a'-'z', '.', '_', '-')+
+SNIPPET_ARGS = "(" SNIPPET_ARG ("," SNIPPET_ARG)* ")"
+SNIPPET_ARG = 
+   "full" |
+   "metric_table" |
+   "omit_requirement_level" |
+   ("tag" "=" ('A'-'Z','a'-'z','0'-'9')+)
+```
+
+
+## Snippet Templates
+
+You can use `weaver_forge` and `minijinja` templates for snippet generation.  When doing so, a template named
+`snippet.md.j2` will be used for all snippet generation.
+
+The template will be passed the following context variables:
+
+- `group`: The resolved semantic convention group, referenced by id in the snippet tag.
+- `snippet_type`: Either `metric_table` or `attribute_table`, based on arguments to the snippet tag.
+- `tag_filter`: The set of all values defined as tag filters.
+- `attribute_registry_base_url`: Base url to use when making attribute registry links.
+
+Otherwise, the template will be given all filters, tests and functions defined in `weaver_forge`.

crates/weaver_semconv_gen/allowed-external-types.toml

@@ -7,5 +7,8 @@ allowed_external_types = [
     "weaver_semconv::path::RegistryPath",
     "weaver_resolver::Error",
     "weaver_cache::Cache",
-    "weaver_common::error::WeaverError"
+    "weaver_cache::Error",
+    "weaver_common::error::WeaverError",
+    "weaver_forge::error::Error",
+    "weaver_forge::TemplateEngine"
 ]

crates/weaver_semconv_gen/data/templates.md

@@ -0,0 +1,7 @@
+<!-- semconv registry.user_agent -->
+attribute_table: registry.user_agent
+<!-- endsemconv -->
+
+<!-- semconv metric.http.server.request.duration(metric_table) -->
+metric_table: metric.http.server.request.duration
+<!-- endsemconv -->