parse group config on groupable nodes (#6965)

parse group config on groupable nodes

.changes/unreleased/Features-20230214-225134.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: parse 'group' config on groupable nodes
+time: 2023-02-14T22:51:34.936228-05:00
+custom:
+  Author: michelleark
+  Issue: "6823"

core/dbt/contracts/graph/manifest.py

@@ -793,8 +793,22 @@ def build_macro_child_map(self):
         forward_edges = build_macro_edges(edge_members)
         return forward_edges
 
+    def build_group_map(self):
+        groupable_nodes = list(
+            chain(
+                self.nodes.values(),
+                self.metrics.values(),
+            )
+        )
+        group_map = {group.name: [] for group in self.groups.values()}
+        for node in groupable_nodes:
+            if node.config.group is not None:
+                group_map[node.config.group].append(node.unique_id)
+        self.group_map = group_map
+
     def writable_manifest(self):
         self.build_parent_and_child_maps()
+        self.build_group_map()
         return WritableManifest(
             nodes=self.nodes,
             sources=self.sources,
@@ -808,6 +822,7 @@ def writable_manifest(self):
             disabled=self.disabled,
             child_map=self.child_map,
             parent_map=self.parent_map,
+            group_map=self.group_map,
         )
 
     def write(self, path):
@@ -1192,6 +1207,11 @@ class WritableManifest(ArtifactMixin):
             description="A mapping from parent nodes to their dependents",
         )
     )
+    group_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from group names to their nodes",
+        )
+    )
     metadata: ManifestMetadata = field(
         metadata=dict(
             description="Metadata about the manifest",

core/dbt/contracts/graph/model_config.py

@@ -366,6 +366,7 @@ def replace(self, **kwargs):
 @dataclass
 class MetricConfig(BaseConfig):
     enabled: bool = True
+    group: Optional[str] = None
 
 
 @dataclass
@@ -403,6 +404,10 @@ class NodeAndTestConfig(BaseConfig):
         default_factory=dict,
         metadata=MergeBehavior.Update.meta(),
     )
+    group: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
 
 
 @dataclass

core/dbt/contracts/graph/nodes.py

@@ -242,6 +242,7 @@ class ParsedNode(NodeInfoMixin, ParsedNodeMandatory, SerializableType):
     description: str = field(default="")
     columns: Dict[str, ColumnInfo] = field(default_factory=dict)
     meta: Dict[str, Any] = field(default_factory=dict)
+    group: Optional[str] = None
     docs: Docs = field(default_factory=Docs)
     patch_path: Optional[str] = None
     build_path: Optional[str] = None
@@ -649,6 +650,7 @@ class GenericTestNode(TestShouldStoreFailures, CompiledNode, HasTestMetadata):
     # Was not able to make mypy happy and keep the code working. We need to
     # refactor the various configs.
     config: TestConfig = field(default_factory=TestConfig)  # type: ignore
+    attached_node: Optional[str] = None
 
     def same_contents(self, other) -> bool:
         if other is None:

core/dbt/contracts/util.py

@@ -278,6 +278,8 @@ def upgrade_manifest_json(manifest: dict) -> dict:
     # add group key
     if "groups" not in manifest:
         manifest["groups"] = {}
+    if "group_map" not in manifest:
+        manifest["group_map"] = {}
     for metric_content in manifest.get("metrics", {}).values():
         # handle attr renames + value translation ("expression" -> "derived")
         metric_content = rename_metric_attr(metric_content)

core/dbt/parser/base.py

@@ -290,6 +290,10 @@ def update_parsed_node_config(
         if "meta" in config_dict and config_dict["meta"]:
             parsed_node.meta = config_dict["meta"]
 
+        # If we have group in the config, copy to node level
+        if "group" in config_dict and config_dict["group"]:
+            parsed_node.group = config_dict["group"]
+
         # If we have docs in the config, merge with the node level, for backwards
         # compatibility with earlier node-only config.
         if "docs" in config_dict and config_dict["docs"]:

core/dbt/parser/manifest.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 import os
 import traceback
-from typing import Dict, Optional, Mapping, Callable, Any, List, Type, Union, Tuple
+from typing import Dict, Optional, Mapping, Callable, Any, List, Type, Union, Tuple, Set
 from itertools import chain
 import time
 from dbt.events.base_types import EventLevel
@@ -253,6 +253,8 @@ def load(self):
             else:
                 # create child_map and parent_map
                 self.saved_manifest.build_parent_and_child_maps()
+                # create group_map
+                self.saved_manifest.build_group_map()
                 # files are different, we need to create a new set of
                 # project_parser_files.
                 try:
@@ -1061,6 +1063,27 @@ def _check_resource_uniqueness(
         alias_resources[full_node_name] = node
 
 
+def _check_valid_group_config(manifest: Manifest):
+    group_names = {group.name for group in manifest.groups.values()}
+
+    for metric in manifest.metrics.values():
+        _check_valid_group_config_node(metric, group_names)
+
+    for node in manifest.nodes.values():
+        _check_valid_group_config_node(node, group_names)
+
+
+def _check_valid_group_config_node(
+    groupable_node: Union[Metric, ManifestNode], valid_group_names: Set[str]
+):
+    groupable_node_group = groupable_node.config.group
+    if groupable_node_group and groupable_node_group not in valid_group_names:
+        raise dbt.exceptions.ParsingError(
+            f"Invalid group '{groupable_node_group}', expected one of {sorted(list(valid_group_names))}",
+            node=groupable_node,
+        )
+
+
 def _warn_for_unused_resource_config_paths(manifest: Manifest, config: RuntimeConfig) -> None:
     resource_fqns: Mapping[str, PathSet] = manifest.get_resource_fqns()
     disabled_fqns: PathSet = frozenset(
@@ -1072,6 +1095,7 @@ def _warn_for_unused_resource_config_paths(manifest: Manifest, config: RuntimeCo
 def _check_manifest(manifest: Manifest, config: RuntimeConfig) -> None:
     _check_resource_uniqueness(manifest, config)
     _warn_for_unused_resource_config_paths(manifest, config)
+    _check_valid_group_config(manifest)
 
 
 def _get_node_column(node, column_name):

core/dbt/parser/partial.py

@@ -934,6 +934,7 @@ def delete_schema_group(self, schema_file, group_dict):
             if unique_id in self.saved_manifest.groups:
                 group = self.saved_manifest.groups[unique_id]
                 if group.name == group_name:
+                    self.schedule_nodes_for_parsing(self.saved_manifest.group_map[group.name])
                     self.deleted_manifest.groups[unique_id] = self.saved_manifest.groups.pop(
                         unique_id
                     )

core/dbt/parser/schemas.py

@@ -35,6 +35,8 @@
     Exposure,
     Metric,
     Group,
+    ManifestNode,
+    GraphMemberNode,
 )
 from dbt.contracts.graph.unparsed import (
     HasColumnDocs,
@@ -346,6 +348,23 @@ def _parse_generic_test(
 
         return node
 
+    def _lookup_attached_node(
+        self, target: Testable
+    ) -> Optional[Union[ManifestNode, GraphMemberNode]]:
+        """Look up attached node for Testable target nodes other than sources. Can be None if generic test attached to SQL node with no corresponding .sql file."""
+        attached_node = None  # type: Optional[Union[ManifestNode, GraphMemberNode]]
+        if not isinstance(target, UnpatchedSourceDefinition):
+            attached_node_unique_id = self.manifest.ref_lookup.get_unique_id(target.name, None)
+            if attached_node_unique_id:
+                attached_node = self.manifest.nodes[attached_node_unique_id]
+            else:
+                disabled_node = self.manifest.disabled_lookup.find(
+                    target.name, None
+                ) or self.manifest.disabled_lookup.find(target.name.upper(), None)
+                if disabled_node:
+                    attached_node = self.manifest.disabled[disabled_node[0].unique_id][0]
+        return attached_node
+
     def store_env_vars(self, target, schema_file_id, env_vars):
         self.manifest.env_vars.update(env_vars)
         if schema_file_id in self.manifest.files:
@@ -408,6 +427,13 @@ def render_test_update(self, node, config, builder, schema_file_id):
                 # we got a ValidationError - probably bad types in config()
                 raise SchemaConfigError(exc, node=node) from exc
 
+        # Set attached_node for generic test nodes, if available.
+        # Generic test node inherits attached node's group config value.
+        attached_node = self._lookup_attached_node(builder.target)
+        if attached_node:
+            node.attached_node = attached_node.unique_id
+            node.group, node.config.group = attached_node.config.group, attached_node.config.group
+
     def parse_node(self, block: GenericTestBlock) -> GenericTestNode:
         """In schema parsing, we rewrite most of the part of parse_node that
         builds the initial node to be parsed, but rendering is basically the
@@ -791,6 +817,7 @@ def get_unparsed_target(self) -> Iterable[NonSourceTarget]:
                     # macros don't have the 'config' key support yet
                     self.normalize_meta_attribute(data, path)
                     self.normalize_docs_attribute(data, path)
+                    self.normalize_group_attribute(data, path)
                 node = self._target_type().from_dict(data)
             except (ValidationError, JSONValidationError) as exc:
                 raise YamlParseDictError(path, self.key, data, exc)
@@ -819,6 +846,9 @@ def normalize_meta_attribute(self, data, path):
     def normalize_docs_attribute(self, data, path):
         return self.normalize_attribute(data, path, "docs")
 
+    def normalize_group_attribute(self, data, path):
+        return self.normalize_attribute(data, path, "group")
+
     def patch_node_config(self, node, patch):
         # Get the ContextConfig that's used in calculating the config
         # This must match the model resource_type that's being patched