dbt run works with Click (#6396)

.changes/unreleased/Under the Hood-20221206-151759.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: '`dbt run` works with Click cli'
+time: 2022-12-06T15:17:59.765623-06:00
+custom:
+  Author: iknox-fa
+  Issue: "5551"
+  PR: "6396"

core/dbt/cli/main.py

@@ -6,10 +6,12 @@
 from dbt.adapters.factory import adapter_management
 from dbt.cli import params as p
 from dbt.cli.flags import Flags
+from dbt.config import RuntimeConfig
 from dbt.config.runtime import load_project, load_profile
 from dbt.events.functions import setup_event_logger
 from dbt.profiler import profiler
 from dbt.task.deps import DepsTask
+from dbt.task.run import RunTask
 from dbt.tracking import initialize_from_flags, track_run
 
 
@@ -109,10 +111,10 @@ def cli(ctx, **kwargs):
 @p.fail_fast
 @p.full_refresh
 @p.indirect_selection
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.show
 @p.state
@@ -155,10 +157,10 @@ def docs(ctx, **kwargs):
 @p.compile_docs
 @p.defer
 @p.exclude
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
 @p.target
@@ -194,11 +196,11 @@ def docs_serve(ctx, **kwargs):
 @p.defer
 @p.exclude
 @p.full_refresh
-@p.models
 @p.parse_only
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
 @p.target
@@ -268,13 +270,13 @@ def init(ctx, **kwargs):
 @click.pass_context
 @p.exclude
 @p.indirect_selection
-@p.models
 @p.output
 @p.output_keys
 @p.profile
 @p.profiles_dir
 @p.project_dir
 @p.resource_type
+@p.select
 @p.selector
 @p.state
 @p.target
@@ -311,10 +313,10 @@ def parse(ctx, **kwargs):
 @p.exclude
 @p.fail_fast
 @p.full_refresh
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
 @p.target
@@ -324,8 +326,13 @@ def parse(ctx, **kwargs):
 @p.version_check
 def run(ctx, **kwargs):
     """Compile SQL and execute against the current target database."""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+
+    config = RuntimeConfig.from_parts(ctx.obj["project"], ctx.obj["profile"], ctx.obj["flags"])
+    task = RunTask(ctx.obj["flags"], config)
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt run operation
@@ -348,10 +355,10 @@ def run_operation(ctx, **kwargs):
 @click.pass_context
 @p.exclude
 @p.full_refresh
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.show
 @p.state
@@ -371,10 +378,10 @@ def seed(ctx, **kwargs):
 @click.pass_context
 @p.defer
 @p.exclude
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
 @p.target
@@ -397,11 +404,11 @@ def source(ctx, **kwargs):
 @source.command("freshness")
 @click.pass_context
 @p.exclude
-@p.models
 @p.output_path  # TODO: Is this ok to re-use?  We have three different output params, how much can we consolidate?
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
 @p.target
@@ -420,10 +427,10 @@ def freshness(ctx, **kwargs):
 @p.exclude
 @p.fail_fast
 @p.indirect_selection
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
 @p.store_failures

core/dbt/cli/params.py

@@ -141,14 +141,6 @@
     hidden=True,
 )
 
-models = click.option(
-    "-m",
-    "-s",
-    "models",
-    envvar=None,
-    help="Specify the nodes to include.",
-    multiple=True,
-)
 
 output = click.option(
     "--output",
@@ -270,6 +262,15 @@
     default="default",
 )
 
+select = click.option(
+    "-m",
+    "-s",
+    "select",
+    envvar=None,
+    help="Specify the nodes to include.",
+    multiple=True,
+)
+
 selector = click.option(
     "--selector", envvar=None, help="The selector name to use, as defined in selectors.yml"
 )

core/dbt/config/runtime.py

@@ -8,7 +8,6 @@
 from .profile import Profile
 from .project import Project
 from .renderer import DbtProjectYamlRenderer, ProfileRenderer
-from .utils import parse_cli_vars
 from dbt import flags
 from dbt.adapters.factory import get_relation_class_by_name, get_include_paths
 from dbt.helper_types import FQNPath, PathSet, DictDefaultEmptyStr
@@ -125,7 +124,7 @@ def from_parts(
             .replace_dict(_project_quoting_dict(project, profile))
         ).to_dict(omit_none=True)
 
-        cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
+        cli_vars: Dict[str, Any] = getattr(args, "vars", {})
 
         return cls(
             project_name=project.project_name,
@@ -234,7 +233,7 @@ def validate(self):
     def collect_parts(cls: Type["RuntimeConfig"], args: Any) -> Tuple[Project, Profile]:
         # profile_name from the project
         project_root = args.project_dir if args.project_dir else os.getcwd()
-        cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
+        cli_vars: Dict[str, Any] = getattr(args, "vars", {})
         profile = cls.get_profile(
             project_root,
             cli_vars,
@@ -541,7 +540,7 @@ def from_parts(
         :param args: The parsed command-line arguments.
         :returns RuntimeConfig: The new configuration.
         """
-        cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
+        cli_vars: Dict[str, Any] = getattr(args, "vars", {})
 
         return cls(
             project_name=project.project_name,

core/dbt/config/utils.py

@@ -12,9 +12,9 @@
 from dbt.exceptions import ValidationException, raise_compiler_error
 
 
-def parse_cli_vars(var_string: str) -> Dict[str, Any]:
+def parse_cli_vars(var: str) -> Dict[str, Any]:
     try:
-        cli_vars = yaml_helper.load_yaml_text(var_string)
+        cli_vars = yaml_helper.load_yaml_text(var)
         var_type = type(cli_vars)
         if var_type is dict:
             return cli_vars

core/dbt/graph/cli.py

@@ -1,5 +1,4 @@
 # special support for CLI argument parsing.
-from dbt import flags
 from copy import deepcopy
 import itertools
 from dbt.clients.yaml_helper import yaml, Loader, Dumper  # noqa: F401
@@ -71,11 +70,14 @@ def parse_union_from_default(
 
 
 def parse_difference(
-    include: Optional[List[str]], exclude: Optional[List[str]]
+    include: Optional[List[str]], exclude: Optional[List[str]], indirect_selection: Any
 ) -> SelectionDifference:
 
+    if include == ():
+        include = None
+
     included = parse_union_from_default(
-        include, DEFAULT_INCLUDES, indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION)
+        include, DEFAULT_INCLUDES, indirect_selection=IndirectSelection(indirect_selection)
     )
     excluded = parse_union_from_default(
         exclude, DEFAULT_EXCLUDES, indirect_selection=IndirectSelection.Eager

core/dbt/task/base.py

@@ -78,8 +78,6 @@ class BaseTask(metaclass=ABCMeta):
     def __init__(self, args, config, project=None):
         self.args = args
         self.config = config
-        if hasattr(config, "args"):
-            self.config.args.single_threaded = False
         self.project = config if isinstance(config, Project) else project
 
     @classmethod

core/dbt/task/debug.py

@@ -71,6 +71,10 @@ def __init__(self, args, config):
             else:
                 self.project_dir = os.getcwd()
         self.project_path = os.path.join(self.project_dir, "dbt_project.yml")
+        # N.B. parse_cli_vars is embedded into the param when using click.
+        # replace this with:
+        # cli_vars: Dict[str, Any] = getattr(args, "vars", {})
+        # when this task is refactored for click
         self.cli_vars = parse_cli_vars(getattr(self.args, "vars", "{}"))
 
         # set by _load_*

core/dbt/task/deps.py

@@ -113,6 +113,10 @@ def from_args(cls, args):
         # into the modules directory
         nearest_project_dir = move_to_nearest_project_dir(args.project_dir)
 
+        # N.B. parse_cli_vars is embedded into the param when using click.
+        # replace this with:
+        # cli_vars: Dict[str, Any] = getattr(args, "vars", {})
+        # when this task is refactored for click
         cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
         project_root: str = args.project_dir or nearest_project_dir
         profile: UnsetProfile = cls._get_unset_profile()

core/dbt/task/run_operation.py

@@ -30,6 +30,11 @@ def _get_macro_parts(self):
         return package_name, macro_name
 
     def _get_kwargs(self) -> Dict[str, Any]:
+        # N.B. parse_cli_vars is embedded into the param when using click.
+        # replace this with:
+        # return self.args.args
+        # when this task is refactored for click
+        # or remove the function completely as it's basically a noop
         return parse_cli_vars(self.args.args)
 
     def compile_manifest(self) -> None:

core/dbt/task/runnable.py

@@ -137,16 +137,23 @@ def exclusion_arg(self):
 
     def get_selection_spec(self) -> SelectionSpec:
         default_selector_name = self.config.get_default_selector_name()
-        if self.args.selector_name:
+        # TODO:  The "eager" string below needs to be replaced with programatic access
+        #  to the default value for the indirect selection parameter in
+        # dbt.cli.params.indirect_selection
+        #
+        # Doing that is actually a little tricky, so I'm punting it to a new ticket GH #6397
+        indirect_selection = getattr(self.args, "INDIRECT_SELECTION", "eager")
+
+        if self.args.selector:
             # use pre-defined selector (--selector)
-            spec = self.config.get_selector(self.args.selector_name)
+            spec = self.config.get_selector(self.args.selector)
         elif not (self.selection_arg or self.exclusion_arg) and default_selector_name:
             # use pre-defined selector (--selector) with default: true
             fire_event(DefaultSelector(name=default_selector_name))
             spec = self.config.get_selector(default_selector_name)
         else:
             # use --select and --exclude args
-            spec = parse_difference(self.selection_arg, self.exclusion_arg)
+            spec = parse_difference(self.selection_arg, self.exclusion_arg, indirect_selection)
         return spec
 
     @abstractmethod

test/unit/test_config.py

@@ -22,7 +22,7 @@
 from dbt.semver import VersionSpecifier
 from dbt.task.run_operation import RunOperationTask
 
-from .utils import normalize, config_from_parts_or_dicts
+from .utils import normalize
 
 INITIAL_ROOT = os.getcwd()
 
@@ -165,7 +165,7 @@ def setUp(self):
             },
             'empty_profile_data': {}
         }
-        self.args = Args(profiles_dir=self.profiles_dir, cli_vars='{}',
+        self.args = Args(profiles_dir=self.profiles_dir, cli_vars={},
                          version_check=True, project_dir=self.project_dir)
         self.env_override = {
             'env_value_type': 'postgres',
@@ -510,7 +510,7 @@ def test_invalid_env_vars(self):
 
     def test_cli_and_env_vars(self):
         self.args.target = 'cli-and-env-vars'
-        self.args.vars = '{"cli_value_host": "cli-postgres-host"}'
+        self.args.vars = {"cli_value_host": "cli-postgres-host"}
         renderer = dbt.config.renderer.ProfileRenderer({'cli_value_host': 'cli-postgres-host'})
         with mock.patch.dict(os.environ, self.env_override):
             profile = self.from_args(renderer=renderer)
@@ -1307,7 +1307,7 @@ def setUp(self):
 
     def test_cli_and_env_vars(self):
         self.args.target = 'cli-and-env-vars'
-        self.args.vars = '{"cli_value_host": "cli-postgres-host", "cli_version": "0.1.2"}'
+        self.args.vars = {"cli_value_host": "cli-postgres-host", "cli_version": "0.1.2"}
         with mock.patch.dict(os.environ, self.env_override), temp_cd(self.project_dir):
             config = dbt.config.RuntimeConfig.from_args(self.args)
 

test/unit/test_graph.py

@@ -300,7 +300,12 @@ def test__dependency_list(self):
         })
         manifest.expect.side_effect = lambda n: MagicMock(unique_id=n)
         selector = NodeSelector(graph, manifest)
-        queue = selector.get_graph_queue(parse_difference(None, None))
+        # TODO:  The "eager" string below needs to be replaced with programatic access
+        #  to the default value for the indirect selection parameter in 
+        # dbt.cli.params.indirect_selection
+        #
+        # Doing that is actually a little tricky, so I'm punting it to a new ticket GH #6397
+        queue = selector.get_graph_queue(parse_difference(None, None, "eager"))
 
         for model_id in model_ids:
             self.assertFalse(queue.empty())

test/unit/test_graph_selection.py

@@ -126,7 +126,12 @@ def test_run_specs(include, exclude, expected):
     graph = _get_graph()
     manifest = _get_manifest(graph)
     selector = graph_selector.NodeSelector(graph, manifest)
-    spec = graph_cli.parse_difference(include, exclude)
+    # TODO:  The "eager" string below needs to be replaced with programatic access
+    #  to the default value for the indirect selection parameter in 
+    # dbt.cli.params.indirect_selection
+    #
+    # Doing that is actually a little tricky, so I'm punting it to a new ticket GH #6397
+    spec = graph_cli.parse_difference(include, exclude, "eager")
     selected, _ = selector.select_nodes(spec)
 
     assert selected == expected

test/unit/test_linker.py

@@ -66,7 +66,12 @@ def assert_would_join(self, queue):
     def _get_graph_queue(self, manifest, include=None, exclude=None):
         graph = compilation.Graph(self.linker.graph)
         selector = NodeSelector(graph, manifest)
-        spec = parse_difference(include, exclude)
+        # TODO:  The "eager" string below needs to be replaced with programatic access
+        #  to the default value for the indirect selection parameter in 
+        # dbt.cli.params.indirect_selection
+        #
+        # Doing that is actually a little tricky, so I'm punting it to a new ticket GH #6397
+        spec = parse_difference(include, exclude, "eager")
         return selector.get_graph_queue(spec)
 
     def test_linker_add_dependency(self):