Merge branch 'main' into microbatch-parallelism

.changes/unreleased/Features-20240416-123234.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Parseable JSON and text output in quiet mode for `dbt show` and `dbt compile`
+time: 2024-04-16T12:32:34.764394-06:00
+custom:
+  Author: dbeatty10
+  Issue: "9840"

.changes/unreleased/Fixes-20241002-142638.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: ' override materialization python models'
+time: 2024-10-02T14:26:38.332191458-03:00
+custom:
+    Author: devmessias
+    Issue: "8520"

.changes/unreleased/Under the Hood-20241104-110509.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Upgrade protobuf
+time: 2024-11-04T11:05:09.433244-05:00
+custom:
+  Author: gshank
+  Issue: "10658"

.github/workflows/main.yml

@@ -59,6 +59,7 @@ jobs:
           python -m pip install --user --upgrade pip
           python -m pip --version
           make dev
+          make dev_req
           mypy --version
           dbt --version
 

.gitignore

@@ -57,6 +57,9 @@ test.env
 makefile.test.env
 *.pytest_cache/
 
+# Unit test artifacts
+index.html
+
 
 # Translations
 *.mo

core/dbt/events/core_types.proto

@@ -1646,6 +1646,7 @@ message ShowNode {
     bool is_inline = 3;
     string output_format = 4;
     string unique_id = 5;
+    bool quiet = 6;
 }
 
 message ShowNodeMsg {
@@ -1660,6 +1661,7 @@ message CompiledNode {
     bool is_inline = 3;
     string output_format = 4;
     string unique_id = 5;
+    bool quiet = 6;
 }
 
 message CompiledNodeMsg {

core/dbt/events/types.py

@@ -1663,7 +1663,9 @@ def message(self) -> str:
                     {"node": self.node_name, "show": json.loads(self.preview)}, indent=2
                 )
         else:
-            if self.is_inline:
+            if self.quiet:
+                return self.preview
+            elif self.is_inline:
                 return f"Previewing inline node:\n{self.preview}"
             else:
                 return f"Previewing node '{self.node_name}':\n{self.preview}"
@@ -1680,7 +1682,9 @@ def message(self) -> str:
             else:
                 return json.dumps({"node": self.node_name, "compiled": self.compiled}, indent=2)
         else:
-            if self.is_inline:
+            if self.quiet:
+                return self.compiled
+            elif self.is_inline:
                 return f"Compiled inline node is:\n{self.compiled}"
             else:
                 return f"Compiled node '{self.node_name}' is:\n{self.compiled}"

core/dbt/parser/base.py

@@ -222,7 +222,6 @@ def _create_parsetime_node(
             name = block.name
         if block.path.relative_path.endswith(".py"):
             language = ModelLanguage.python
-            config.add_config_call({"materialized": "table"})
         else:
             # this is not ideal but we have a lot of tests to adjust if don't do it
             language = ModelLanguage.sql
@@ -322,6 +321,15 @@ def update_parsed_node_config(
         # build_config_dict takes the config_call_dict in the ContextConfig object
         # and calls calculate_node_config to combine dbt_project configs and
         # config calls from SQL files, plus patch configs (from schema files)
+        # This normalize the config for a model node due #8520; should be improved latter
+        if not patch_config_dict:
+            patch_config_dict = {}
+        if (
+            parsed_node.resource_type == NodeType.Model
+            and parsed_node.language == ModelLanguage.python
+        ):
+            if "materialized" not in patch_config_dict:
+                patch_config_dict["materialized"] = "table"
         config_dict = config.build_config_dict(patch_config_dict=patch_config_dict)
 
         # Set tags on node provided in config blocks. Tags are additive, so even if

core/dbt/task/compile.py

@@ -4,6 +4,7 @@
 from dbt.artifacts.schemas.run import RunResult, RunStatus
 from dbt.contracts.graph.manifest import Manifest
 from dbt.events.types import CompiledNode, ParseInlineNodeError
+from dbt.flags import get_flags
 from dbt.graph import ResourceTypeSelector
 from dbt.node_types import EXECUTABLE_NODE_TYPES, NodeType
 from dbt.parser.manifest import process_node
@@ -96,6 +97,7 @@ def task_end_messages(self, results) -> None:
                     is_inline=is_inline,
                     output_format=output_format,
                     unique_id=result.node.unique_id,
+                    quiet=get_flags().QUIET,
                 )
             )
 

core/dbt/task/show.py

@@ -7,6 +7,7 @@
 from dbt.context.providers import generate_runtime_model_context
 from dbt.contracts.graph.nodes import SeedNode
 from dbt.events.types import ShowNode
+from dbt.flags import get_flags
 from dbt.task.base import ConfiguredTask
 from dbt.task.compile import CompileRunner, CompileTask
 from dbt.task.seed import SeedRunner
@@ -108,6 +109,7 @@ def task_end_messages(self, results) -> None:
                     is_inline=is_inline,
                     output_format=self.args.output,
                     unique_id=result.node.unique_id,
+                    quiet=get_flags().QUIET,
                 )
             )
 
@@ -143,5 +145,6 @@ def run(self):
                     is_inline=True,
                     output_format=self.args.output,
                     unique_id="direct-query",
+                    quiet=get_flags().QUIET,
                 )
             )

core/setup.py

@@ -57,7 +57,7 @@
         # with major versions in each new minor version of dbt-core.
         "click>=8.0.2,<9.0",
         "networkx>=2.3,<4.0",
-        "protobuf>=4.0.0,<5",
+        "protobuf>=5.0,<6.0",
         "requests<3.0.0",  # should match dbt-common
         "snowplow-tracker>=1.0.2,<2.0",
         # ----
@@ -71,7 +71,7 @@
         "dbt-extractor>=0.5.0,<=0.6",
         "dbt-semantic-interfaces>=0.7.4,<0.8",
         # Minor versions for these are expected to be backwards-compatible
-        "dbt-common>=1.11.0,<2.0",
+        "dbt-common>=1.13.0,<2.0",
         "dbt-adapters>=1.9.0,<2.0",
         # ----
         # Expect compatibility with all new versions of these packages, so lower bounds only.

dev-requirements.txt

@@ -19,6 +19,7 @@ isort==5.13.2
 mypy==1.4.1
 pip-tools
 pre-commit
+protobuf>=5.0,<6.0
 pytest>=7.4,<8.0
 pytest-cov
 pytest-csv>=3.0,<4.0
@@ -32,7 +33,7 @@ types-docutils
 types-PyYAML
 types-Jinja2
 types-mock
-types-protobuf>=4.0.0,<5.0.0
+types-protobuf>=5.0,<6.0
 types-pytz
 types-requests
 types-setuptools

tests/functional/compile/test_compile.py

@@ -169,11 +169,25 @@ def test_inline_pass(self, project):
         assert len(results) == 1
         assert "Compiled inline node is:" in log_output
 
+    def test_inline_pass_quiet(self, project):
+        (results, log_output) = run_dbt_and_capture(
+            ["compile", "--quiet", "--inline", "select * from {{ ref('first_model') }}"]
+        )
+        assert len(results) == 1
+        assert "Compiled inline node is:" not in log_output
+
     def test_select_pass(self, project):
         (results, log_output) = run_dbt_and_capture(["compile", "--select", "second_model"])
         assert len(results) == 3
         assert "Compiled node 'second_model' is:" in log_output
 
+    def test_select_pass_quiet(self, project):
+        (results, log_output) = run_dbt_and_capture(
+            ["compile", "--quiet", "--select", "second_model"]
+        )
+        assert len(results) == 3
+        assert "Compiled node 'second_model' is:" not in log_output
+
     def test_select_pass_empty(self, project):
         (results, log_output) = run_dbt_and_capture(
             ["compile", "--indirect-selection", "empty", "--select", "second_model"]
@@ -201,6 +215,17 @@ def test_output_json_select(self, project):
         assert len(results) == 3
         assert "node" in log_output
         assert "compiled" in log_output
+        with pytest.raises(json.JSONDecodeError):
+            json.loads(log_output)
+
+    def test_output_json_select_quiet(self, project):
+        (results, log_output) = run_dbt_and_capture(
+            ["compile", "--quiet", "--select", "second_model", "--output", "json"]
+        )
+        assert len(results) == 3
+        assert "node" in log_output
+        assert "compiled" in log_output
+        json.loads(log_output)
 
     def test_output_json_inline(self, project):
         (results, log_output) = run_dbt_and_capture(
@@ -209,6 +234,24 @@ def test_output_json_inline(self, project):
         assert len(results) == 1
         assert '"node"' not in log_output
         assert '"compiled"' in log_output
+        with pytest.raises(json.JSONDecodeError):
+            json.loads(log_output)
+
+    def test_output_json_inline_quiet(self, project):
+        (results, log_output) = run_dbt_and_capture(
+            [
+                "compile",
+                "--quiet",
+                "--inline",
+                "select * from {{ ref('second_model') }}",
+                "--output",
+                "json",
+            ]
+        )
+        assert len(results) == 1
+        assert '"node"' not in log_output
+        assert '"compiled"' in log_output
+        json.loads(log_output)
 
     def test_compile_inline_not_add_node(self, project):
         dbt = dbtTestRunner()

tests/functional/show/test_show.py

@@ -1,3 +1,5 @@
+import json
+
 import pytest
 
 from dbt.tests.util import run_dbt, run_dbt_and_capture
@@ -73,6 +75,18 @@ def test_select_single_model_json(self, project):
         assert "Previewing node 'sample_model'" not in log_output
         assert "sample_num" in log_output
         assert "sample_bool" in log_output
+        with pytest.raises(json.JSONDecodeError):
+            json.loads(log_output)
+
+    def test_select_single_model_json_quiet(self, project):
+        run_dbt(["build"])
+        (_, log_output) = run_dbt_and_capture(
+            ["show", "--quiet", "--select", "sample_model", "--output", "json"]
+        )
+        assert "Previewing node 'sample_model'" not in log_output
+        assert "sample_num" in log_output
+        assert "sample_bool" in log_output
+        json.loads(log_output)
 
 
 class TestShowNumeric(ShowBase):
@@ -119,6 +133,15 @@ def test_inline_pass(self, project):
         assert "sample_num" in log_output
         assert "sample_bool" in log_output
 
+    def test_inline_pass_quiet(self, project):
+        run_dbt(["build"])
+        (_, log_output) = run_dbt_and_capture(
+            ["show", "--quiet", "--inline", "select * from {{ ref('sample_model') }}"]
+        )
+        assert "Previewing inline node" not in log_output
+        assert "sample_num" in log_output
+        assert "sample_bool" in log_output
+
 
 class TestShowInlineFail(ShowBase):
     def test_inline_fail(self, project):
@@ -148,6 +171,16 @@ def test_inline_direct_pass(self, project):
         # which will load the adapter fully and satisfy the teardown code.
         run_dbt(["seed"])
 
+    def test_inline_direct_pass_quiet(self, project):
+        query = f"select * from {project.test_schema}.sample_seed"
+        (_, log_output) = run_dbt_and_capture(["show", "--quiet", "--inline-direct", query])
+        assert "Previewing inline node" not in log_output
+        assert "sample_num" in log_output
+        assert "sample_bool" in log_output
+
+        # See prior test for explanation of why this is here
+        run_dbt(["seed"])
+
 
 class TestShowInlineDirectFail(ShowBase):
 

tests/unit/parser/test_parser.py

@@ -1005,7 +1005,7 @@ def model(dbt, session):
 import pandas as pd
 
 def model(dbt, session):
-    dbt.config(materialized="view")
+    dbt.config(materialized="incremental")
     return pd.dataframe([1, 2])
 """
 
@@ -1200,7 +1200,7 @@ def test_python_model_custom_materialization(self):
         self.parser.manifest.files[block.file.file_id] = block.file
         self.parser.parse_file(block)
         node = list(self.parser.manifest.nodes.values())[0]
-        self.assertEqual(node.get_materialization(), "view")
+        self.assertEqual(node.get_materialization(), "incremental")
 
 
 class StaticModelParserTest(BaseParserTest):