From ef914a0e3f53d7f1a480684680fd7fb806c006b7 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Fri, 15 Jul 2022 13:47:09 -0500 Subject: [PATCH 01/21] CLI modeling RAT, commands and global params --- core/dbt/cli/README.md | 1 + core/dbt/cli/__init__.py | 0 core/dbt/cli/global_params.py | 114 ++++++++++++++++++++ core/dbt/cli/main.py | 191 ++++++++++++++++++++++++++++++++++ core/setup.py | 4 +- 5 files changed, 307 insertions(+), 3 deletions(-) create mode 100644 core/dbt/cli/README.md create mode 100644 core/dbt/cli/__init__.py create mode 100644 core/dbt/cli/global_params.py create mode 100644 core/dbt/cli/main.py diff --git a/core/dbt/cli/README.md b/core/dbt/cli/README.md new file mode 100644 index 00000000000..1333ed77b7e --- /dev/null +++ b/core/dbt/cli/README.md @@ -0,0 +1 @@ +TODO diff --git a/core/dbt/cli/__init__.py b/core/dbt/cli/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/cli/global_params.py b/core/dbt/cli/global_params.py new file mode 100644 index 00000000000..753834496d1 --- /dev/null +++ b/core/dbt/cli/global_params.py @@ -0,0 +1,114 @@ +import click +from os import path + +# Commonly used global options (sort alphabeticaly) +profiles_dir = click.option( + "--profiles-dir", + help="Which directory to look in for the profiles.yml file. Default = " + + path.join(path.expanduser("~"), ".dbt"), + default=path.join(path.expanduser("~"), ".dbt"), +) + +version = click.option("--version", help="Show version information", is_flag=True, default=False) + +# Global options that override config (sort alphabeticaly) +cache_selected_only = click.option( + "--cache-selected-only/--no-cache-selected-only", + help="Pre cache database objects relevant to selected resource only.", + default=False, +) + +debug = click.option( + "--debug/--no-debug", + "-d/ ", + help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", + default=False, +) + +fail_fast = click.option( + "--fail-fast/--no-fail-fast", "-x/ ", help="Stop execution on first failure.", default=False +) + +log_format = click.option( + "--log-format", + help="Specify the log format, overriding the command's default.", + type=click.Choice(["text", "json", "default"], case_sensitive=False), + default="default", +) + +partial_parse = click.option( + "--partial-parse/--no-partial-parse", + help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.", + default=True, +) + +print = click.option( + "--print/--no-print", help="Output all {{ print() }} macro calls.", default=True +) + +printer_width = click.option( + "--printer_width", help="Sets the width of terminal output", type=click.INT, default=80 +) + +quiet = click.option( + "--quiet/--no-quiet", + help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", + default=False, +) + +send_anonymous_usage_stats = click.option( + "--anonymous-usage-stats/--no-anonymous-usage-stats", + help="Send anonymous usage stats to dbt Labs.", + default=True, +) + +static_parser = click.option( + "--static-parser/--no-static-parser", help="Use the static parser.", default=True +) + +use_colors = click.option( + "--use-colors/--no-use-colors", + help="Output is colorized by default and may also be set in a profile or at the command line.", + default=True, +) + +use_experimental_parser = click.option( + "--use-experimental-parser/--no-use-experimental-parser", + help="Enable experimental parsing features.", + default=False, +) + +version_check = click.option( + "--version-check/--no-version-check", + help="Ensure dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version')", + default=True, +) + +warn_error = click.option( + "--warn-error/--no-warn-error", + help="If dbt would normally warn, instead raise an exception. Examples include --models that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", + default=False, +) + +write_json = click.option( + "--write-json/--no-write-json", + help="Writing the manifest and run_results.json files to disk", + default=True, +) + + +# Rarely used global options +event_buffer_size = click.option( + "--event-buffer-size", + help="Sets the max number of events to buffer in EVENT_HISTORY.", + default=100000, + type=click.INT, +) + +record_timing = click.option( + "-r", + "--record-timing-info", + help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", + is_flag=True, + default=False, +) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py new file mode 100644 index 00000000000..1fcbdab2c53 --- /dev/null +++ b/core/dbt/cli/main.py @@ -0,0 +1,191 @@ +import click +import dbt.cli.global_params as global_params +import inspect # This is temporary for RAT-ing +import sys + + +# dbt +@click.group( + invoke_without_command=True, + no_args_is_help=True, + epilog="Specify one of these sub-commands and you can find more help from there.", +) +@click.pass_context +@global_params.profiles_dir +@global_params.version +@global_params.cache_selected_only +@global_params.debug +@global_params.fail_fast +@global_params.log_format +@global_params.partial_parse +@global_params.print +@global_params.printer_width +@global_params.quiet +@global_params.send_anonymous_usage_stats +@global_params.static_parser +@global_params.use_colors +@global_params.use_experimental_parser +@global_params.version_check +@global_params.warn_error +@global_params.write_json +@global_params.event_buffer_size +@global_params.record_timing +def cli(ctx, **kwargs): + """An ELT tool for managing your SQL transformations and data models. + For more documentation on these commands, visit: docs.getdbt.com + """ + + if kwargs.get("version", False): + click.echo(f"`version` called\n ctx.params: {ctx.params}") + sys.exit() + else: + del ctx.params["version"] + + +# dbt build +@cli.command("build") +@click.pass_context +def build(ctx, **kwargs): + """Run all Seeds, Models, Snapshots, and tests in DAG order""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt clean +@cli.command("clean") +@click.pass_context +def clean(ctx, **kwargs): + """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt docs +@cli.group() +@click.pass_context +def docs(ctx, **kwargs): + """Generate or serve the documentation website for your project""" + + +# dbt docs generate +@docs.command("generate") +@click.pass_context +def docs_generate(ctx, **kwargs): + """Generate the documentation website for your project""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + ) + + +# dbt docs serve +@docs.command("serve") +@click.pass_context +def docs_serve(ctx, **kwargs): + """Serve the documentation website for your project""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + ) + + +# dbt compile +@cli.command("compile") +@click.pass_context +def compile(ctx, **kwargs): + """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt debug +@cli.command("debug") +@click.pass_context +def debug(ctx, **kwargs): + """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt deps +@cli.command("deps") +@click.pass_context +def deps(ctx, **kwargs): + """Pull the most recent version of the dependencies listed in packages.yml""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt init +@cli.command("init") +@click.pass_context +def init(ctx, **kwargs): + """Initialize a new DBT project.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt list +# dbt TODO: Figure out aliasing for ls (or just c/p?) +@cli.command("list") +@click.pass_context +def list(ctx, **kwargs): + """List the resources in your project""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt parse +@cli.command("parse") +@click.pass_context +def parse(ctx, **kwargs): + """Parses the project and provides information on performance""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt run +@cli.command("run") +@click.pass_context +def run(ctx, **kwargs): + """Compile SQL and execute against the current target database.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt run operation +@cli.command("run-operation") +@click.pass_context +def run_operation(ctx, **kwargs): + """Run the named macro with any supplied arguments.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt seed +@cli.command("seed") +@click.pass_context +def seed(ctx, **kwargs): + """Load data from csv files into your data warehouse.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt snapshot +@cli.command("snapshot") +@click.pass_context +def snapshot(ctx, **kwargs): + """Execute snapshots defined in your project""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt source +@cli.group() +@click.pass_context +def source(ctx, **kwargs): + """Manage your project's sources""" + + +# dbt source freshness +@source.command("freshness") +@click.pass_context +def freshness(ctx, **kwargs): + """Snapshots the current freshness of the project's sources""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + ) + + +# dbt test +@cli.command("test") +@click.pass_context +def test(ctx, **kwargs): + """Runs tests on data in deployed models. Run this after `dbt run`""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") diff --git a/core/setup.py b/core/setup.py index 776a71897a7..5e0a5bde2e4 100644 --- a/core/setup.py +++ b/core/setup.py @@ -43,9 +43,7 @@ include_package_data=True, test_suite="test", entry_points={ - "console_scripts": [ - "dbt = dbt.main:main", - ], + "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:cli"], }, install_requires=[ "Jinja2==2.11.3", From 4476158eb6c16d0cbd0c43c161075719103ff514 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Fri, 15 Jul 2022 14:02:41 -0500 Subject: [PATCH 02/21] pretty output --- core/dbt/cli/main.py | 61 ++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 1fcbdab2c53..08a5c28502c 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,6 +1,7 @@ import click import dbt.cli.global_params as global_params import inspect # This is temporary for RAT-ing +from pprint import pformat as pf import sys @@ -36,7 +37,7 @@ def cli(ctx, **kwargs): """ if kwargs.get("version", False): - click.echo(f"`version` called\n ctx.params: {ctx.params}") + click.echo(f"`version` called\n ctx.params: {pf(ctx.params)}") sys.exit() else: del ctx.params["version"] @@ -47,7 +48,9 @@ def cli(ctx, **kwargs): @click.pass_context def build(ctx, **kwargs): """Run all Seeds, Models, Snapshots, and tests in DAG order""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt clean @@ -55,7 +58,9 @@ def build(ctx, **kwargs): @click.pass_context def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt docs @@ -71,7 +76,7 @@ def docs(ctx, **kwargs): def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" ) @@ -81,7 +86,7 @@ def docs_generate(ctx, **kwargs): def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" ) @@ -90,7 +95,9 @@ def docs_serve(ctx, **kwargs): @click.pass_context def compile(ctx, **kwargs): """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt debug @@ -98,7 +105,9 @@ def compile(ctx, **kwargs): @click.pass_context def debug(ctx, **kwargs): """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt deps @@ -106,7 +115,9 @@ def debug(ctx, **kwargs): @click.pass_context def deps(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt init @@ -114,7 +125,9 @@ def deps(ctx, **kwargs): @click.pass_context def init(ctx, **kwargs): """Initialize a new DBT project.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt list @@ -123,7 +136,9 @@ def init(ctx, **kwargs): @click.pass_context def list(ctx, **kwargs): """List the resources in your project""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt parse @@ -131,7 +146,9 @@ def list(ctx, **kwargs): @click.pass_context def parse(ctx, **kwargs): """Parses the project and provides information on performance""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt run @@ -139,7 +156,9 @@ def parse(ctx, **kwargs): @click.pass_context def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt run operation @@ -147,7 +166,9 @@ def run(ctx, **kwargs): @click.pass_context def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt seed @@ -155,7 +176,9 @@ def run_operation(ctx, **kwargs): @click.pass_context def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt snapshot @@ -163,7 +186,9 @@ def seed(ctx, **kwargs): @click.pass_context def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt source @@ -179,7 +204,7 @@ def source(ctx, **kwargs): def freshness(ctx, **kwargs): """Snapshots the current freshness of the project's sources""" click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" ) @@ -188,4 +213,6 @@ def freshness(ctx, **kwargs): @click.pass_context def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) From 3d36e8a6923f25011c1ff6e7f63e0aa4864a40aa Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Tue, 19 Jul 2022 14:18:12 -0500 Subject: [PATCH 03/21] updates --- core/dbt/cli/main.py | 10 ++++-- core/dbt/cli/params/__init__.py | 0 .../{global_params.py => params/_global.py} | 10 +----- core/dbt/cli/params/project.py | 31 +++++++++++++++++++ 4 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 core/dbt/cli/params/__init__.py rename core/dbt/cli/{global_params.py => params/_global.py} (91%) create mode 100644 core/dbt/cli/params/project.py diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 08a5c28502c..fef5e09c10e 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,5 +1,6 @@ import click -import dbt.cli.global_params as global_params +from dbt.cli.params import _global as global_params +from dbt.cli.params import project as project_params import inspect # This is temporary for RAT-ing from pprint import pformat as pf import sys @@ -12,7 +13,6 @@ epilog="Specify one of these sub-commands and you can find more help from there.", ) @click.pass_context -@global_params.profiles_dir @global_params.version @global_params.cache_selected_only @global_params.debug @@ -35,7 +35,6 @@ def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com """ - if kwargs.get("version", False): click.echo(f"`version` called\n ctx.params: {pf(ctx.params)}") sys.exit() @@ -56,6 +55,11 @@ def build(ctx, **kwargs): # dbt clean @cli.command("clean") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" click.echo( diff --git a/core/dbt/cli/params/__init__.py b/core/dbt/cli/params/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/cli/global_params.py b/core/dbt/cli/params/_global.py similarity index 91% rename from core/dbt/cli/global_params.py rename to core/dbt/cli/params/_global.py index 753834496d1..7ae3d07c501 100644 --- a/core/dbt/cli/global_params.py +++ b/core/dbt/cli/params/_global.py @@ -1,14 +1,6 @@ import click -from os import path - -# Commonly used global options (sort alphabeticaly) -profiles_dir = click.option( - "--profiles-dir", - help="Which directory to look in for the profiles.yml file. Default = " - + path.join(path.expanduser("~"), ".dbt"), - default=path.join(path.expanduser("~"), ".dbt"), -) +# Version is a special snowflake, list it first version = click.option("--version", help="Show version information", is_flag=True, default=False) # Global options that override config (sort alphabeticaly) diff --git a/core/dbt/cli/params/project.py b/core/dbt/cli/params/project.py new file mode 100644 index 00000000000..1be04952068 --- /dev/null +++ b/core/dbt/cli/params/project.py @@ -0,0 +1,31 @@ +import click +from pathlib import Path, PurePath + +project_dir = click.option( + "--project-dir", + help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", + default=Path.cwd(), + type=click.Path(exists=True), +) + +profiles_dir = click.option( + "--profiles-dir", + help=f"Which directory to look in for the profiles.yml file. Default = {PurePath.joinpath(Path.home(), '.dbt')}", + default=PurePath.joinpath(Path.home(), ".dbt"), + type=click.Path( + exists=True, + ), +) + +profile = click.option( + "--profile", + help="Which profile to load. Overrides setting in dbt_project.yml.", +) + +target = click.option("-t", "--target", help="Which target to load for the given profile") + +# TODO validate the yaml (validation callback + custom type) +vars = click.option( + "--vars", + help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", +) From 03af44502a9b264d2c91d293bb767ad3ecf5ccb5 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Tue, 19 Jul 2022 15:10:28 -0500 Subject: [PATCH 04/21] added multi-flag example and a custom type --- core/dbt/cli/main.py | 17 ++++++++++++++ core/dbt/cli/params/project.py | 2 ++ core/dbt/cli/params/run.py | 43 ++++++++++++++++++++++++++++++++++ core/dbt/cli/params/types.py | 17 ++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 core/dbt/cli/params/run.py create mode 100644 core/dbt/cli/params/types.py diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index fef5e09c10e..276590dc8f7 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,6 +1,7 @@ import click from dbt.cli.params import _global as global_params from dbt.cli.params import project as project_params +from dbt.cli.params import run as run_params import inspect # This is temporary for RAT-ing from pprint import pformat as pf import sys @@ -158,6 +159,22 @@ def parse(ctx, **kwargs): # dbt run @cli.command("run") @click.pass_context +@global_params.fail_fast +@global_params.version_check +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.log_path +@run_params.target_path +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@run_params.defer +@run_params.full_refresh def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" click.echo( diff --git a/core/dbt/cli/params/project.py b/core/dbt/cli/params/project.py index 1be04952068..8e10fc8eea3 100644 --- a/core/dbt/cli/params/project.py +++ b/core/dbt/cli/params/project.py @@ -1,5 +1,6 @@ import click from pathlib import Path, PurePath +from dbt.cli.params.types import YAML project_dir = click.option( "--project-dir", @@ -28,4 +29,5 @@ vars = click.option( "--vars", help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), ) diff --git a/core/dbt/cli/params/run.py b/core/dbt/cli/params/run.py new file mode 100644 index 00000000000..ccaae943637 --- /dev/null +++ b/core/dbt/cli/params/run.py @@ -0,0 +1,43 @@ +import click + +log_path = click.option( + "--log-path", + help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", + type=click.Path(), +) + +target_path = click.option( + "--target-path", + help="Configure the 'target-path'. Only applies this setting for the current run. Overrides the 'DBT_TARGET_PATH' if it is set.", + type=click.Path(), +) + +threads = click.option( + "--threads", + help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", + default=1, + type=click.INT, +) + +models = click.option("-m", "-s", help="Specify the nodes to include.", multiple=True) + +exclude = click.option("--exclude", help="Specify the nodes to exclude.") + +selector = click.option("--selector", help="The selector name to use, as defined in selectors.yml") + +state = click.option( + "--state", + help="If set, use the given directory as the source for json files to compare with this project.", +) + +defer = click.option( + "--defer/--no-defer", + help="If set, defer to the state variable for resolving unselected nodes.", + default=True, +) + +full_refresh = click.option( + "--full_refresh", + help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", + is_flag=True, +) diff --git a/core/dbt/cli/params/types.py b/core/dbt/cli/params/types.py new file mode 100644 index 00000000000..6c024b4966b --- /dev/null +++ b/core/dbt/cli/params/types.py @@ -0,0 +1,17 @@ +from click import ParamType +import yaml + + +class YAML(ParamType): + """The YAML type converts YAML strings into objects.""" + + name = "YAML" + + def convert(self, value, param, ctx): + # assume non-string values are a problem + if not isinstance(value, str): + self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) + try: + return yaml.load(value, Loader=yaml.Loader) + except yaml.parser.ParserError: + self.fail(f"String '{value}' is not valid YAML", param, ctx) From 83f84ef206c3690b88096f0a7fcba8fa41796732 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 11 Aug 2022 14:23:14 -0500 Subject: [PATCH 05/21] feature complete --- core/dbt/cli/main.py | 147 ++++++++++++++++++++++++++++++++- core/dbt/cli/params/docs.py | 17 ++++ core/dbt/cli/params/misc.py | 46 +++++++++++ core/dbt/cli/params/parse.py | 13 +++ core/dbt/cli/params/project.py | 1 - core/dbt/cli/params/run.py | 43 +++++++++- 6 files changed, 263 insertions(+), 4 deletions(-) create mode 100644 core/dbt/cli/params/docs.py create mode 100644 core/dbt/cli/params/misc.py create mode 100644 core/dbt/cli/params/parse.py diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 276590dc8f7..b7de2eaaccd 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,5 +1,8 @@ import click from dbt.cli.params import _global as global_params +from dbt.cli.params import docs as docs_params +from dbt.cli.params import misc as misc_params +from dbt.cli.params import parse as parse_params from dbt.cli.params import project as project_params from dbt.cli.params import run as run_params import inspect # This is temporary for RAT-ing @@ -56,9 +59,9 @@ def build(ctx, **kwargs): # dbt clean @cli.command("clean") @click.pass_context -@project_params.profile -@project_params.profiles_dir @project_params.project_dir +@project_params.profiles_dir +@project_params.profile @project_params.target @project_params.vars def clean(ctx, **kwargs): @@ -78,6 +81,21 @@ def docs(ctx, **kwargs): # dbt docs generate @docs.command("generate") @click.pass_context +@global_params.version_check +@project_params.project_dir +@project_params.profiles_dir +@project_params.profile +@project_params.target +@project_params.vars +@docs_params.compile +@run_params.defer +@run_params.threads +@run_params.target_path +@run_params.log_path +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" click.echo( @@ -88,6 +106,13 @@ def docs_generate(ctx, **kwargs): # dbt docs serve @docs.command("serve") @click.pass_context +@project_params.project_dir +@project_params.profiles_dir +@project_params.profile +@project_params.target +@project_params.vars +@docs_params.port +@docs_params.browser def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" click.echo( @@ -98,6 +123,22 @@ def docs_serve(ctx, **kwargs): # dbt compile @cli.command("compile") @click.pass_context +@global_params.version_check +@project_params.project_dir +@project_params.profiles_dir +@project_params.profile +@project_params.target +@project_params.vars +@run_params.parse_only +@run_params.threads +@run_params.target_path +@run_params.log_path +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@run_params.defer +@run_params.full_refresh def compile(ctx, **kwargs): """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" click.echo( @@ -108,6 +149,13 @@ def compile(ctx, **kwargs): # dbt debug @cli.command("debug") @click.pass_context +@global_params.version_check +@project_params.project_dir +@project_params.profiles_dir +@project_params.profile +@project_params.target +@project_params.vars +@misc_params.config_dir def debug(ctx, **kwargs): """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" click.echo( @@ -118,6 +166,11 @@ def debug(ctx, **kwargs): # dbt deps @cli.command("deps") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars def deps(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml""" click.echo( @@ -128,6 +181,12 @@ def deps(ctx, **kwargs): # dbt init @cli.command("init") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@misc_params.skip_profile_setup def init(ctx, **kwargs): """Initialize a new DBT project.""" click.echo( @@ -139,6 +198,19 @@ def init(ctx, **kwargs): # dbt TODO: Figure out aliasing for ls (or just c/p?) @cli.command("list") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@misc_params.output +@misc_params.ouptut_keys +@run_params.resource_type +@run_params.models +@run_params.indirect_selection +@run_params.exclude +@run_params.selector +@run_params.state def list(ctx, **kwargs): """List the resources in your project""" click.echo( @@ -149,6 +221,17 @@ def list(ctx, **kwargs): # dbt parse @cli.command("parse") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@parse_params.write_manifest +@parse_params.compile +@run_params.threads +@run_params.target_path +@run_params.log_path +@global_params.version_check def parse(ctx, **kwargs): """Parses the project and provides information on performance""" click.echo( @@ -185,6 +268,12 @@ def run(ctx, **kwargs): # dbt run operation @cli.command("run-operation") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.args def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" click.echo( @@ -195,6 +284,21 @@ def run_operation(ctx, **kwargs): # dbt seed @cli.command("seed") @click.pass_context +@global_params.version_check +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.full_refresh +@run_params.log_path +@run_params.target_path +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@misc_params.show def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" click.echo( @@ -205,6 +309,17 @@ def seed(ctx, **kwargs): # dbt snapshot @cli.command("snapshot") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@run_params.defer def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" click.echo( @@ -222,6 +337,17 @@ def source(ctx, **kwargs): # dbt source freshness @source.command("freshness") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@misc_params.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? def freshness(ctx, **kwargs): """Snapshots the current freshness of the project's sources""" click.echo( @@ -232,6 +358,23 @@ def freshness(ctx, **kwargs): # dbt test @cli.command("test") @click.pass_context +@global_params.fail_fast +@global_params.version_check +@misc_params.store_failures +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.indirect_selection +@run_params.log_path +@run_params.target_path +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@run_params.defer def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" click.echo( diff --git a/core/dbt/cli/params/docs.py b/core/dbt/cli/params/docs.py new file mode 100644 index 00000000000..05425824dc3 --- /dev/null +++ b/core/dbt/cli/params/docs.py @@ -0,0 +1,17 @@ +import click + +compile = click.option( + "--compile/--no-compile", + help="Wether or not to run 'dbt compile' as part of docs generation", + default=True, +) + +port = click.option( + "--port", help="Specify the port number for the docs server", default=8080, type=click.INT +) + +browser = click.option( + "--browser/--no-browser", + help="Wether or not to open a local web browser after starting the server", + default=True, +) diff --git a/core/dbt/cli/params/misc.py b/core/dbt/cli/params/misc.py new file mode 100644 index 00000000000..6e85d3d2760 --- /dev/null +++ b/core/dbt/cli/params/misc.py @@ -0,0 +1,46 @@ +import click +from pathlib import Path, PurePath + +config_dir = click.option( + "--config-dir", + help="If specified, DBT will show path information for this project", + type=click.STRING, +) + +skip_profile_setup = click.option( + "--skip-profile-setup", + "-s", + help="Skip interative profile setup.", + default=False, +) + +output = click.option( + "--output", + help="TODO: No current help text", + type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), + default="name", +) + +ouptut_keys = click.option( + "--output-keys", + help="TODO: No current help text", + default=False, +) + +show = click.option( + "--show", + help="Show a sample of the loaded data in the terminal", + default=False, +) + +output_path = click.option( + "--output", + "-o", + help="Specify the output path for the json report. By default, outputs to 'target/sources.json'", + type=click.Path(file_okay=True, dir_okay=False, writable=True), + default=PurePath.joinpath(Path.cwd(), "target/sources.json"), +) + +store_failures = click.option( + "--store-failures", help="Store test results (failing rows) in the database", default=False +) diff --git a/core/dbt/cli/params/parse.py b/core/dbt/cli/params/parse.py new file mode 100644 index 00000000000..0f376d91e31 --- /dev/null +++ b/core/dbt/cli/params/parse.py @@ -0,0 +1,13 @@ +import click + +compile = click.option( + "--compile/--no-compile", + help="TODO: No help text currently available", + default=True, +) + +write_manifest = click.option( + "--write-manifest/--no-write-manifest", + help="TODO: No help text currently available", + default=True, +) diff --git a/core/dbt/cli/params/project.py b/core/dbt/cli/params/project.py index 8e10fc8eea3..00a3494732f 100644 --- a/core/dbt/cli/params/project.py +++ b/core/dbt/cli/params/project.py @@ -25,7 +25,6 @@ target = click.option("-t", "--target", help="Which target to load for the given profile") -# TODO validate the yaml (validation callback + custom type) vars = click.option( "--vars", help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", diff --git a/core/dbt/cli/params/run.py b/core/dbt/cli/params/run.py index ccaae943637..035bfe06f75 100644 --- a/core/dbt/cli/params/run.py +++ b/core/dbt/cli/params/run.py @@ -1,4 +1,5 @@ import click +from dbt.cli.params.types import YAML log_path = click.option( "--log-path", @@ -37,7 +38,47 @@ ) full_refresh = click.option( - "--full_refresh", + "--full-refresh", help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", is_flag=True, ) + +parse_only = click.option( + "--parse-only", + help="TODO: No help text currently available", + is_flag=True, +) + +resource_type = click.option( + "--resource-type", + help="TODO: No current help text", + type=click.Choice( + [ + "metric", + "source", + "analysis", + "model", + "test", + "exposure", + "snapshot", + "seed", + "default", + "all", + ], + case_sensitive=False, + ), + default="default", +) + +indirect_selection = click.option( + "--indirect_selection", + help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.", + type=click.Choice(["eager", "cautious"], case_sensitive=False), + default="eager", +) + +args = click.option( + "--args", + help="Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the selected macro. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), +) From 61650cafd91c11556351fb125dbfbef3d7db48b4 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 11 Aug 2022 16:04:25 -0500 Subject: [PATCH 06/21] consolidate params --- core/dbt/cli/main.py | 371 ++++++++++++++++---------------- core/dbt/cli/params.py | 304 ++++++++++++++++++++++++++ core/dbt/cli/params/__init__.py | 0 core/dbt/cli/params/_global.py | 106 --------- core/dbt/cli/params/docs.py | 17 -- core/dbt/cli/params/misc.py | 46 ---- core/dbt/cli/params/parse.py | 13 -- core/dbt/cli/params/project.py | 32 --- core/dbt/cli/params/run.py | 84 -------- core/dbt/cli/params/types.py | 17 -- 10 files changed, 488 insertions(+), 502 deletions(-) create mode 100644 core/dbt/cli/params.py delete mode 100644 core/dbt/cli/params/__init__.py delete mode 100644 core/dbt/cli/params/_global.py delete mode 100644 core/dbt/cli/params/docs.py delete mode 100644 core/dbt/cli/params/misc.py delete mode 100644 core/dbt/cli/params/parse.py delete mode 100644 core/dbt/cli/params/project.py delete mode 100644 core/dbt/cli/params/run.py delete mode 100644 core/dbt/cli/params/types.py diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index b7de2eaaccd..3f11c001827 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,14 +1,11 @@ import click -from dbt.cli.params import _global as global_params -from dbt.cli.params import docs as docs_params -from dbt.cli.params import misc as misc_params -from dbt.cli.params import parse as parse_params -from dbt.cli.params import project as project_params -from dbt.cli.params import run as run_params -import inspect # This is temporary for RAT-ing -from pprint import pformat as pf +from dbt.cli import params as p import sys +# This is temporary for RAT-ing +import inspect +from pprint import pformat as pf + # dbt @click.group( @@ -17,24 +14,24 @@ epilog="Specify one of these sub-commands and you can find more help from there.", ) @click.pass_context -@global_params.version -@global_params.cache_selected_only -@global_params.debug -@global_params.fail_fast -@global_params.log_format -@global_params.partial_parse -@global_params.print -@global_params.printer_width -@global_params.quiet -@global_params.send_anonymous_usage_stats -@global_params.static_parser -@global_params.use_colors -@global_params.use_experimental_parser -@global_params.version_check -@global_params.warn_error -@global_params.write_json -@global_params.event_buffer_size -@global_params.record_timing +@p.version +@p.cache_selected_only +@p.debug +@p.fail_fast +@p.log_format +@p.partial_parse +@p.print +@p.printer_width +@p.quiet +@p.send_anonymous_usage_stats +@p.static_parser +@p.use_colors +@p.use_experimental_parser +@p.version_check +@p.warn_error +@p.write_json +@p.event_buffer_size +@p.record_timing def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com @@ -59,11 +56,11 @@ def build(ctx, **kwargs): # dbt clean @cli.command("clean") @click.pass_context -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" click.echo( @@ -81,21 +78,21 @@ def docs(ctx, **kwargs): # dbt docs generate @docs.command("generate") @click.pass_context -@global_params.version_check -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars -@docs_params.compile -@run_params.defer -@run_params.threads -@run_params.target_path -@run_params.log_path -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state +@p.version_check +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.compile_docs +@p.defer +@p.threads +@p.target_path +@p.log_path +@p.models +@p.exclude +@p.selector +@p.state def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" click.echo( @@ -106,13 +103,13 @@ def docs_generate(ctx, **kwargs): # dbt docs serve @docs.command("serve") @click.pass_context -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars -@docs_params.port -@docs_params.browser +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.port +@p.browser def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" click.echo( @@ -123,22 +120,22 @@ def docs_serve(ctx, **kwargs): # dbt compile @cli.command("compile") @click.pass_context -@global_params.version_check -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars -@run_params.parse_only -@run_params.threads -@run_params.target_path -@run_params.log_path -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@run_params.defer -@run_params.full_refresh +@p.version_check +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.parse_only +@p.threads +@p.target_path +@p.log_path +@p.models +@p.exclude +@p.selector +@p.state +@p.defer +@p.full_refresh def compile(ctx, **kwargs): """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" click.echo( @@ -149,13 +146,13 @@ def compile(ctx, **kwargs): # dbt debug @cli.command("debug") @click.pass_context -@global_params.version_check -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars -@misc_params.config_dir +@p.version_check +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.config_dir def debug(ctx, **kwargs): """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" click.echo( @@ -166,11 +163,11 @@ def debug(ctx, **kwargs): # dbt deps @cli.command("deps") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars def deps(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml""" click.echo( @@ -181,12 +178,12 @@ def deps(ctx, **kwargs): # dbt init @cli.command("init") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@misc_params.skip_profile_setup +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.skip_profile_setup def init(ctx, **kwargs): """Initialize a new DBT project.""" click.echo( @@ -198,19 +195,19 @@ def init(ctx, **kwargs): # dbt TODO: Figure out aliasing for ls (or just c/p?) @cli.command("list") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@misc_params.output -@misc_params.ouptut_keys -@run_params.resource_type -@run_params.models -@run_params.indirect_selection -@run_params.exclude -@run_params.selector -@run_params.state +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.output +@p.ouptut_keys +@p.resource_type +@p.models +@p.indirect_selection +@p.exclude +@p.selector +@p.state def list(ctx, **kwargs): """List the resources in your project""" click.echo( @@ -221,17 +218,17 @@ def list(ctx, **kwargs): # dbt parse @cli.command("parse") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@parse_params.write_manifest -@parse_params.compile -@run_params.threads -@run_params.target_path -@run_params.log_path -@global_params.version_check +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.write_manifest +@p.compile_parse +@p.threads +@p.target_path +@p.log_path +@p.version_check def parse(ctx, **kwargs): """Parses the project and provides information on performance""" click.echo( @@ -242,22 +239,22 @@ def parse(ctx, **kwargs): # dbt run @cli.command("run") @click.pass_context -@global_params.fail_fast -@global_params.version_check -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.log_path -@run_params.target_path -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@run_params.defer -@run_params.full_refresh +@p.fail_fast +@p.version_check +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.log_path +@p.target_path +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.defer +@p.full_refresh def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" click.echo( @@ -268,12 +265,12 @@ def run(ctx, **kwargs): # dbt run operation @cli.command("run-operation") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.args +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.args def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" click.echo( @@ -284,21 +281,21 @@ def run_operation(ctx, **kwargs): # dbt seed @cli.command("seed") @click.pass_context -@global_params.version_check -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.full_refresh -@run_params.log_path -@run_params.target_path -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@misc_params.show +@p.version_check +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.full_refresh +@p.log_path +@p.target_path +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.show def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" click.echo( @@ -309,17 +306,17 @@ def seed(ctx, **kwargs): # dbt snapshot @cli.command("snapshot") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@run_params.defer +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.defer def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" click.echo( @@ -337,17 +334,17 @@ def source(ctx, **kwargs): # dbt source freshness @source.command("freshness") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@misc_params.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? def freshness(ctx, **kwargs): """Snapshots the current freshness of the project's sources""" click.echo( @@ -358,23 +355,23 @@ def freshness(ctx, **kwargs): # dbt test @cli.command("test") @click.pass_context -@global_params.fail_fast -@global_params.version_check -@misc_params.store_failures -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.indirect_selection -@run_params.log_path -@run_params.target_path -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@run_params.defer +@p.fail_fast +@p.version_check +@p.store_failures +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.indirect_selection +@p.log_path +@p.target_path +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.defer def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" click.echo( diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py new file mode 100644 index 00000000000..f991359ecf7 --- /dev/null +++ b/core/dbt/cli/params.py @@ -0,0 +1,304 @@ +import click +import yaml +from pathlib import Path, PurePath +from click import ParamType + + +class YAML(ParamType): + """The Click YAML type. Converts YAML strings into objects.""" + + name = "YAML" + + def convert(self, value, param, ctx): + # assume non-string values are a problem + if not isinstance(value, str): + self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) + try: + return yaml.load(value, Loader=yaml.Loader) + except yaml.parser.ParserError: + self.fail(f"String '{value}' is not valid YAML", param, ctx) + + +args = click.option( + "--args", + help="Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the selected macro. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), +) + +browser = click.option( + "--browser/--no-browser", + help="Wether or not to open a local web browser after starting the server", + default=True, +) + +cache_selected_only = click.option( + "--cache-selected-only/--no-cache-selected-only", + help="Pre cache database objects relevant to selected resource only.", + default=False, +) + +compile_docs = click.option( + "--compile/--no-compile", + help="Wether or not to run 'dbt compile' as part of docs generation", + default=True, +) + +compile_parse = click.option( + "--compile/--no-compile", + help="TODO: No help text currently available", + default=True, +) + +config_dir = click.option( + "--config-dir", + help="If specified, DBT will show path information for this project", + type=click.STRING, +) + +debug = click.option( + "--debug/--no-debug", + "-d/ ", + help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", + default=False, +) + +defer = click.option( + "--defer/--no-defer", + help="If set, defer to the state variable for resolving unselected nodes.", + default=True, +) + +event_buffer_size = click.option( + "--event-buffer-size", + help="Sets the max number of events to buffer in EVENT_HISTORY.", + default=100000, + type=click.INT, +) + +exclude = click.option("--exclude", help="Specify the nodes to exclude.") + +fail_fast = click.option( + "--fail-fast/--no-fail-fast", "-x/ ", help="Stop execution on first failure.", default=False +) + +full_refresh = click.option( + "--full-refresh", + help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", + is_flag=True, +) + +indirect_selection = click.option( + "--indirect_selection", + help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.", + type=click.Choice(["eager", "cautious"], case_sensitive=False), + default="eager", +) + +log_format = click.option( + "--log-format", + help="Specify the log format, overriding the command's default.", + type=click.Choice(["text", "json", "default"], case_sensitive=False), + default="default", +) + +log_path = click.option( + "--log-path", + help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", + type=click.Path(), +) + +models = click.option("-m", "-s", help="Specify the nodes to include.", multiple=True) + +output = click.option( + "--output", + help="TODO: No current help text", + type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), + default="name", +) + +ouptut_keys = click.option( + "--output-keys", + help="TODO: No current help text", + default=False, +) + +output_path = click.option( + "--output", + "-o", + help="Specify the output path for the json report. By default, outputs to 'target/sources.json'", + type=click.Path(file_okay=True, dir_okay=False, writable=True), + default=PurePath.joinpath(Path.cwd(), "target/sources.json"), +) + +parse_only = click.option( + "--parse-only", + help="TODO: No help text currently available", + is_flag=True, +) + +partial_parse = click.option( + "--partial-parse/--no-partial-parse", + help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.", + default=True, +) + +port = click.option( + "--port", help="Specify the port number for the docs server", default=8080, type=click.INT +) + +print = click.option( + "--print/--no-print", help="Output all {{ print() }} macro calls.", default=True +) + +printer_width = click.option( + "--printer_width", help="Sets the width of terminal output", type=click.INT, default=80 +) + +profile = click.option( + "--profile", + help="Which profile to load. Overrides setting in dbt_project.yml.", +) + +profiles_dir = click.option( + "--profiles-dir", + help=f"Which directory to look in for the profiles.yml file. Default = {PurePath.joinpath(Path.home(), '.dbt')}", + default=PurePath.joinpath(Path.home(), ".dbt"), + type=click.Path( + exists=True, + ), +) + +project_dir = click.option( + "--project-dir", + help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", + default=Path.cwd(), + type=click.Path(exists=True), +) + +quiet = click.option( + "--quiet/--no-quiet", + help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", + default=False, +) + +record_timing = click.option( + "-r", + "--record-timing-info", + help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", + is_flag=True, + default=False, +) + +resource_type = click.option( + "--resource-type", + help="TODO: No current help text", + type=click.Choice( + [ + "metric", + "source", + "analysis", + "model", + "test", + "exposure", + "snapshot", + "seed", + "default", + "all", + ], + case_sensitive=False, + ), + default="default", +) + +selector = click.option("--selector", help="The selector name to use, as defined in selectors.yml") + +send_anonymous_usage_stats = click.option( + "--anonymous-usage-stats/--no-anonymous-usage-stats", + help="Send anonymous usage stats to dbt Labs.", + default=True, +) + +show = click.option( + "--show", + help="Show a sample of the loaded data in the terminal", + default=False, +) + +skip_profile_setup = click.option( + "--skip-profile-setup", + "-s", + help="Skip interative profile setup.", + default=False, +) + +state = click.option( + "--state", + help="If set, use the given directory as the source for json files to compare with this project.", +) + +static_parser = click.option( + "--static-parser/--no-static-parser", help="Use the static parser.", default=True +) + +store_failures = click.option( + "--store-failures", help="Store test results (failing rows) in the database", default=False +) + +target = click.option("-t", "--target", help="Which target to load for the given profile") + +target_path = click.option( + "--target-path", + help="Configure the 'target-path'. Only applies this setting for the current run. Overrides the 'DBT_TARGET_PATH' if it is set.", + type=click.Path(), +) + +threads = click.option( + "--threads", + help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", + default=1, + type=click.INT, +) + +use_colors = click.option( + "--use-colors/--no-use-colors", + help="Output is colorized by default and may also be set in a profile or at the command line.", + default=True, +) + +use_experimental_parser = click.option( + "--use-experimental-parser/--no-use-experimental-parser", + help="Enable experimental parsing features.", + default=False, +) + +vars = click.option( + "--vars", + help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), +) + +version = click.option("--version", help="Show version information", is_flag=True, default=False) + +version_check = click.option( + "--version-check/--no-version-check", + help="Ensure dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version')", + default=True, +) + +warn_error = click.option( + "--warn-error/--no-warn-error", + help="If dbt would normally warn, instead raise an exception. Examples include --models that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", + default=False, +) + +write_json = click.option( + "--write-json/--no-write-json", + help="Writing the manifest and run_results.json files to disk", + default=True, +) + +write_manifest = click.option( + "--write-manifest/--no-write-manifest", + help="TODO: No help text currently available", + default=True, +) diff --git a/core/dbt/cli/params/__init__.py b/core/dbt/cli/params/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/core/dbt/cli/params/_global.py b/core/dbt/cli/params/_global.py deleted file mode 100644 index 7ae3d07c501..00000000000 --- a/core/dbt/cli/params/_global.py +++ /dev/null @@ -1,106 +0,0 @@ -import click - -# Version is a special snowflake, list it first -version = click.option("--version", help="Show version information", is_flag=True, default=False) - -# Global options that override config (sort alphabeticaly) -cache_selected_only = click.option( - "--cache-selected-only/--no-cache-selected-only", - help="Pre cache database objects relevant to selected resource only.", - default=False, -) - -debug = click.option( - "--debug/--no-debug", - "-d/ ", - help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", - default=False, -) - -fail_fast = click.option( - "--fail-fast/--no-fail-fast", "-x/ ", help="Stop execution on first failure.", default=False -) - -log_format = click.option( - "--log-format", - help="Specify the log format, overriding the command's default.", - type=click.Choice(["text", "json", "default"], case_sensitive=False), - default="default", -) - -partial_parse = click.option( - "--partial-parse/--no-partial-parse", - help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.", - default=True, -) - -print = click.option( - "--print/--no-print", help="Output all {{ print() }} macro calls.", default=True -) - -printer_width = click.option( - "--printer_width", help="Sets the width of terminal output", type=click.INT, default=80 -) - -quiet = click.option( - "--quiet/--no-quiet", - help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", - default=False, -) - -send_anonymous_usage_stats = click.option( - "--anonymous-usage-stats/--no-anonymous-usage-stats", - help="Send anonymous usage stats to dbt Labs.", - default=True, -) - -static_parser = click.option( - "--static-parser/--no-static-parser", help="Use the static parser.", default=True -) - -use_colors = click.option( - "--use-colors/--no-use-colors", - help="Output is colorized by default and may also be set in a profile or at the command line.", - default=True, -) - -use_experimental_parser = click.option( - "--use-experimental-parser/--no-use-experimental-parser", - help="Enable experimental parsing features.", - default=False, -) - -version_check = click.option( - "--version-check/--no-version-check", - help="Ensure dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version')", - default=True, -) - -warn_error = click.option( - "--warn-error/--no-warn-error", - help="If dbt would normally warn, instead raise an exception. Examples include --models that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", - default=False, -) - -write_json = click.option( - "--write-json/--no-write-json", - help="Writing the manifest and run_results.json files to disk", - default=True, -) - - -# Rarely used global options -event_buffer_size = click.option( - "--event-buffer-size", - help="Sets the max number of events to buffer in EVENT_HISTORY.", - default=100000, - type=click.INT, -) - -record_timing = click.option( - "-r", - "--record-timing-info", - help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", - is_flag=True, - default=False, -) diff --git a/core/dbt/cli/params/docs.py b/core/dbt/cli/params/docs.py deleted file mode 100644 index 05425824dc3..00000000000 --- a/core/dbt/cli/params/docs.py +++ /dev/null @@ -1,17 +0,0 @@ -import click - -compile = click.option( - "--compile/--no-compile", - help="Wether or not to run 'dbt compile' as part of docs generation", - default=True, -) - -port = click.option( - "--port", help="Specify the port number for the docs server", default=8080, type=click.INT -) - -browser = click.option( - "--browser/--no-browser", - help="Wether or not to open a local web browser after starting the server", - default=True, -) diff --git a/core/dbt/cli/params/misc.py b/core/dbt/cli/params/misc.py deleted file mode 100644 index 6e85d3d2760..00000000000 --- a/core/dbt/cli/params/misc.py +++ /dev/null @@ -1,46 +0,0 @@ -import click -from pathlib import Path, PurePath - -config_dir = click.option( - "--config-dir", - help="If specified, DBT will show path information for this project", - type=click.STRING, -) - -skip_profile_setup = click.option( - "--skip-profile-setup", - "-s", - help="Skip interative profile setup.", - default=False, -) - -output = click.option( - "--output", - help="TODO: No current help text", - type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), - default="name", -) - -ouptut_keys = click.option( - "--output-keys", - help="TODO: No current help text", - default=False, -) - -show = click.option( - "--show", - help="Show a sample of the loaded data in the terminal", - default=False, -) - -output_path = click.option( - "--output", - "-o", - help="Specify the output path for the json report. By default, outputs to 'target/sources.json'", - type=click.Path(file_okay=True, dir_okay=False, writable=True), - default=PurePath.joinpath(Path.cwd(), "target/sources.json"), -) - -store_failures = click.option( - "--store-failures", help="Store test results (failing rows) in the database", default=False -) diff --git a/core/dbt/cli/params/parse.py b/core/dbt/cli/params/parse.py deleted file mode 100644 index 0f376d91e31..00000000000 --- a/core/dbt/cli/params/parse.py +++ /dev/null @@ -1,13 +0,0 @@ -import click - -compile = click.option( - "--compile/--no-compile", - help="TODO: No help text currently available", - default=True, -) - -write_manifest = click.option( - "--write-manifest/--no-write-manifest", - help="TODO: No help text currently available", - default=True, -) diff --git a/core/dbt/cli/params/project.py b/core/dbt/cli/params/project.py deleted file mode 100644 index 00a3494732f..00000000000 --- a/core/dbt/cli/params/project.py +++ /dev/null @@ -1,32 +0,0 @@ -import click -from pathlib import Path, PurePath -from dbt.cli.params.types import YAML - -project_dir = click.option( - "--project-dir", - help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", - default=Path.cwd(), - type=click.Path(exists=True), -) - -profiles_dir = click.option( - "--profiles-dir", - help=f"Which directory to look in for the profiles.yml file. Default = {PurePath.joinpath(Path.home(), '.dbt')}", - default=PurePath.joinpath(Path.home(), ".dbt"), - type=click.Path( - exists=True, - ), -) - -profile = click.option( - "--profile", - help="Which profile to load. Overrides setting in dbt_project.yml.", -) - -target = click.option("-t", "--target", help="Which target to load for the given profile") - -vars = click.option( - "--vars", - help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", - type=YAML(), -) diff --git a/core/dbt/cli/params/run.py b/core/dbt/cli/params/run.py deleted file mode 100644 index 035bfe06f75..00000000000 --- a/core/dbt/cli/params/run.py +++ /dev/null @@ -1,84 +0,0 @@ -import click -from dbt.cli.params.types import YAML - -log_path = click.option( - "--log-path", - help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", - type=click.Path(), -) - -target_path = click.option( - "--target-path", - help="Configure the 'target-path'. Only applies this setting for the current run. Overrides the 'DBT_TARGET_PATH' if it is set.", - type=click.Path(), -) - -threads = click.option( - "--threads", - help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", - default=1, - type=click.INT, -) - -models = click.option("-m", "-s", help="Specify the nodes to include.", multiple=True) - -exclude = click.option("--exclude", help="Specify the nodes to exclude.") - -selector = click.option("--selector", help="The selector name to use, as defined in selectors.yml") - -state = click.option( - "--state", - help="If set, use the given directory as the source for json files to compare with this project.", -) - -defer = click.option( - "--defer/--no-defer", - help="If set, defer to the state variable for resolving unselected nodes.", - default=True, -) - -full_refresh = click.option( - "--full-refresh", - help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", - is_flag=True, -) - -parse_only = click.option( - "--parse-only", - help="TODO: No help text currently available", - is_flag=True, -) - -resource_type = click.option( - "--resource-type", - help="TODO: No current help text", - type=click.Choice( - [ - "metric", - "source", - "analysis", - "model", - "test", - "exposure", - "snapshot", - "seed", - "default", - "all", - ], - case_sensitive=False, - ), - default="default", -) - -indirect_selection = click.option( - "--indirect_selection", - help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.", - type=click.Choice(["eager", "cautious"], case_sensitive=False), - default="eager", -) - -args = click.option( - "--args", - help="Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the selected macro. This argument should be a YAML string, eg. '{my_variable: my_value}'", - type=YAML(), -) diff --git a/core/dbt/cli/params/types.py b/core/dbt/cli/params/types.py deleted file mode 100644 index 6c024b4966b..00000000000 --- a/core/dbt/cli/params/types.py +++ /dev/null @@ -1,17 +0,0 @@ -from click import ParamType -import yaml - - -class YAML(ParamType): - """The YAML type converts YAML strings into objects.""" - - name = "YAML" - - def convert(self, value, param, ctx): - # assume non-string values are a problem - if not isinstance(value, str): - self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) - try: - return yaml.load(value, Loader=yaml.Loader) - except yaml.parser.ParserError: - self.fail(f"String '{value}' is not valid YAML", param, ctx) From 421472656563efd3fb740b22a1e18d17f98701c1 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 11 Aug 2022 18:10:32 -0500 Subject: [PATCH 07/21] tests --- tests/unit/test_cli.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/unit/test_cli.py diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 00000000000..294961c98e7 --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,25 @@ +from dbt.cli.main import cli +import click + + +class TestCLI: + def test_commands_have_docstrings(self): + def run_test(commands): + for _, command in commands.items(): + if type(command) is click.core.Command: + assert command.__doc__ is not None + if type(command) is click.core.Group: + run_test(command.commands) + + run_test(cli.commands) + + def test_params_have_help_texts(self): + def run_test(commands): + for _, command in commands.items(): + if type(command) is click.core.Command: + for param in command.params: + assert param.help is not None + if type(command) is click.core.Group: + run_test(command.commands) + + run_test(cli.commands) From 95e36c3c87a8cad3bc83a7b870f445218eab43ac Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Tue, 23 Aug 2022 16:04:14 -0500 Subject: [PATCH 08/21] Auto-env-var approach --- core/dbt/cli/main.py | 4 ++++ core/setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 3f11c001827..e5fe6d4442f 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -377,3 +377,7 @@ def test(ctx, **kwargs): click.echo( f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" ) + + +def runner(): + cli(auto_envvar_prefix="DBT") diff --git a/core/setup.py b/core/setup.py index 2c254dcf6b0..6ac908eaa51 100644 --- a/core/setup.py +++ b/core/setup.py @@ -43,7 +43,7 @@ include_package_data=True, test_suite="test", entry_points={ - "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:cli"], + "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:runner"], }, install_requires=[ "Jinja2==3.1.2", From cac3e24e637c1f4abca5210b1056d16dde48b8dc Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Wed, 7 Sep 2022 10:40:33 -0500 Subject: [PATCH 09/21] mostly complete --- core/dbt/cli/flags.py | 42 ++++++++++ core/dbt/cli/main.py | 23 +++--- core/dbt/cli/option_types.py | 33 ++++++++ core/dbt/cli/params.py | 153 +++++++++++++++++++++++++++-------- core/setup.py | 2 +- tests/unit/test_cli.py | 41 +++++++--- 6 files changed, 234 insertions(+), 60 deletions(-) create mode 100644 core/dbt/cli/flags.py create mode 100644 core/dbt/cli/option_types.py diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py new file mode 100644 index 00000000000..7c305cd77e3 --- /dev/null +++ b/core/dbt/cli/flags.py @@ -0,0 +1,42 @@ +# TODO Move this to /core/dbt/flags.py when we're ready to break things +import os +from dataclasses import dataclass +from multiprocessing import get_context + +from click import get_current_context + +if os.name != "nt": + # https://bugs.python.org/issue41567 + import multiprocessing.popen_spawn_posix # type: ignore # noqa: F401 + + +@dataclass(frozen=True) +class Flags: + def __init__(self) -> None: + + ctx = get_current_context() + + def assign_params(ctx): + """Recursively adds all click params to flag object""" + for param_name, param_value in ctx.params.items(): + # N.B. You have to use the base MRO method (object.__setattr__) to set attributes + # when using frozen dataclasses. + # https://docs.python.org/3/library/dataclasses.html#frozen-instances + if hasattr(self, param_name): + raise Exception(f"Duplicate flag names found in click command: {param_name}") + object.__setattr__(self, param_name.upper(), param_value) + if ctx.parent: + assign_params(ctx.parent) + + assign_params(ctx) + + # Hard coded flags + # TODO: This is listed as for backwards compat... Still needed? + object.__setattr__(self, "STRICT_MODE", False) + # TODO: Appears unused... Still needed? + object.__setattr__(self, "WHICH", None) + object.__setattr__(self, "MP_CONTEXT", get_context("spawn")) + + # Support console DO NOT TRACK initiave + if os.getenv("DO_NOT_TRACK", "").lower() in (1, "t", "true", "y", "yes"): + object.__setattr__(self, "ANONYMOUS_USAGE_STATS", False) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index e5fe6d4442f..a9a261720f2 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,29 +1,33 @@ -import click -from dbt.cli import params as p +import inspect # This is temporary for RAT-ing import sys +from pprint import pformat as pf # This is temporary for RAT-ing -# This is temporary for RAT-ing -import inspect -from pprint import pformat as pf +import click +from dbt.cli import params as p +from dbt.cli.flags import Flags # dbt @click.group( + context_settings={"help_option_names": ["-h", "--help"]}, invoke_without_command=True, no_args_is_help=True, epilog="Specify one of these sub-commands and you can find more help from there.", ) @click.pass_context +@p.anonymous_usage_stats @p.version @p.cache_selected_only @p.debug +@p.enable_legacy_logger @p.fail_fast +@p.log_cache_events @p.log_format +@p.macro_debugging @p.partial_parse @p.print @p.printer_width @p.quiet -@p.send_anonymous_usage_stats @p.static_parser @p.use_colors @p.use_experimental_parser @@ -63,8 +67,9 @@ def build(ctx, **kwargs): @p.vars def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" + flags = Flags() click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}\n flags {flags}" ) @@ -377,7 +382,3 @@ def test(ctx, **kwargs): click.echo( f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" ) - - -def runner(): - cli(auto_envvar_prefix="DBT") diff --git a/core/dbt/cli/option_types.py b/core/dbt/cli/option_types.py new file mode 100644 index 00000000000..523df651775 --- /dev/null +++ b/core/dbt/cli/option_types.py @@ -0,0 +1,33 @@ +from click import ParamType +import yaml + + +class YAML(ParamType): + """The Click YAML type. Converts YAML strings into objects.""" + + name = "YAML" + + def convert(self, value, param, ctx): + # assume non-string values are a problem + if not isinstance(value, str): + self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) + try: + return yaml.load(value, Loader=yaml.Loader) + except yaml.parser.ParserError: + self.fail(f"String '{value}' is not valid YAML", param, ctx) + + +class Truthy(ParamType): + """The Click Truthy type. Converts strings into a "truthy" type""" + + name = "TRUTHY" + + def convert(self, value, param, ctx): + # assume non-string / non-None values are a problem + if not isinstance(value, (str, None)): + self.fail(f"Cannot load TRUTHY from type {type(value)}", param, ctx) + + if value is None or value.lower() in ("0", "false", "f"): + return None + else: + return value diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index f991359ecf7..76bd4c5e434 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -1,56 +1,53 @@ -import click -import yaml from pathlib import Path, PurePath -from click import ParamType - - -class YAML(ParamType): - """The Click YAML type. Converts YAML strings into objects.""" - - name = "YAML" - def convert(self, value, param, ctx): - # assume non-string values are a problem - if not isinstance(value, str): - self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) - try: - return yaml.load(value, Loader=yaml.Loader) - except yaml.parser.ParserError: - self.fail(f"String '{value}' is not valid YAML", param, ctx) +import click +from dbt.cli.option_types import YAML, Truthy +anonymous_usage_stats = click.option( + "--anonymous-usage-stats/--no-anonymous-usage-stats", + envvar="DBT_ANONYMOUS_USAGE_STATS", + help="Send anonymous usage stats to dbt Labs.", + default=True, +) args = click.option( "--args", + envvar=None, help="Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the selected macro. This argument should be a YAML string, eg. '{my_variable: my_value}'", type=YAML(), ) browser = click.option( "--browser/--no-browser", + envvar=None, help="Wether or not to open a local web browser after starting the server", default=True, ) cache_selected_only = click.option( "--cache-selected-only/--no-cache-selected-only", + envvar="DBT_CACHE_SELECTED_ONLY", help="Pre cache database objects relevant to selected resource only.", default=False, ) compile_docs = click.option( "--compile/--no-compile", + envvar=None, help="Wether or not to run 'dbt compile' as part of docs generation", default=True, ) compile_parse = click.option( "--compile/--no-compile", + envvar=None, help="TODO: No help text currently available", default=True, ) config_dir = click.option( "--config-dir", + envvar=None, help="If specified, DBT will show path information for this project", type=click.STRING, ) @@ -58,44 +55,73 @@ def convert(self, value, param, ctx): debug = click.option( "--debug/--no-debug", "-d/ ", + envvar="DBT_DEBUG", help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", default=False, ) +# TODO: The env var and name (reflected in flags) are corrections! +# The original name was `DEFER_MODE` and used an env var called "DBT_DEFER_TO_STATE" +# Both of which break existing naming conventions. +# This will need to be fixed before use in the main codebase and communicated as a change to the community! defer = click.option( "--defer/--no-defer", + envvar="DBT_DEFER", help="If set, defer to the state variable for resolving unselected nodes.", default=True, ) +enable_legacy_logger = click.option( + "--enable-legacy-logger/--no-enable-legacy-logger", + envvar="DBT_ENABLE_LEGACY_LOGGER", + hidden=True, + default=False, +) + event_buffer_size = click.option( "--event-buffer-size", + envvar="DBT_EVENT_BUFFER_SIZE", help="Sets the max number of events to buffer in EVENT_HISTORY.", default=100000, type=click.INT, ) -exclude = click.option("--exclude", help="Specify the nodes to exclude.") +exclude = click.option("--exclude", envvar=None, help="Specify the nodes to exclude.") fail_fast = click.option( - "--fail-fast/--no-fail-fast", "-x/ ", help="Stop execution on first failure.", default=False + "--fail-fast/--no-fail-fast", + "-x/ ", + envvar="DBT_FAIL_FAST", + help="Stop execution on first failure.", + default=False, ) full_refresh = click.option( "--full-refresh", + envvar="DBT_FULL_REFRESH", help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", is_flag=True, ) indirect_selection = click.option( - "--indirect_selection", + "--indirect-selection", + envvar="DBT_INDIRECT_SELECTION", help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.", type=click.Choice(["eager", "cautious"], case_sensitive=False), default="eager", ) +# TODO: Should this be exposed in the help text? +log_cache_events = click.option( + "--log-cache-events/--no-log-cache-events", + envvar="DBT_LOG_CACHE_EVENTS", + hidden=True, + default=None, +) + log_format = click.option( "--log-format", + envvar="DBT_LOG_FORMAT", help="Specify the log format, overriding the command's default.", type=click.Choice(["text", "json", "default"], case_sensitive=False), default="default", @@ -103,14 +129,23 @@ def convert(self, value, param, ctx): log_path = click.option( "--log-path", + envvar="DBT_LOG_PATH", help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", type=click.Path(), ) -models = click.option("-m", "-s", help="Specify the nodes to include.", multiple=True) +macro_debugging = click.option( + "--macro-debugging/--no-macro-debugging", + envvar="DBT_MACRO_DEBUGGING", + hidden=True, + default=None, +) + +models = click.option("-m", "-s", envvar=None, help="Specify the nodes to include.", multiple=True) output = click.option( "--output", + envvar=None, help="TODO: No current help text", type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), default="name", @@ -118,6 +153,7 @@ def convert(self, value, param, ctx): ouptut_keys = click.option( "--output-keys", + envvar=None, help="TODO: No current help text", default=False, ) @@ -125,6 +161,7 @@ def convert(self, value, param, ctx): output_path = click.option( "--output", "-o", + envvar=None, help="Specify the output path for the json report. By default, outputs to 'target/sources.json'", type=click.Path(file_okay=True, dir_okay=False, writable=True), default=PurePath.joinpath(Path.cwd(), "target/sources.json"), @@ -132,35 +169,54 @@ def convert(self, value, param, ctx): parse_only = click.option( "--parse-only", + envvar=None, help="TODO: No help text currently available", is_flag=True, ) partial_parse = click.option( "--partial-parse/--no-partial-parse", + envvar="DBT_PARTIAL_PARSE", help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.", default=True, ) port = click.option( - "--port", help="Specify the port number for the docs server", default=8080, type=click.INT + "--port", + envvar=None, + help="Specify the port number for the docs server", + default=8080, + type=click.INT, ) +# TODO: The env var and name (reflected in flags) are corrections! +# The original name was `NO_PRINT` and used the env var `DBT_NO_PRINT`. +# Both of which break existing naming conventions. +# This will need to be fixed before use in the main codebase and communicated as a change to the community! print = click.option( - "--print/--no-print", help="Output all {{ print() }} macro calls.", default=True + "--print/--no-print", + envvar="DBT_PRINT", + help="Output all {{ print() }} macro calls.", + default=True, ) printer_width = click.option( - "--printer_width", help="Sets the width of terminal output", type=click.INT, default=80 + "--printer-width", + envvar="DBT_PRINTER_WIDTH", + help="Sets the width of terminal output", + type=click.INT, + default=80, ) profile = click.option( "--profile", + envvar=None, help="Which profile to load. Overrides setting in dbt_project.yml.", ) profiles_dir = click.option( "--profiles-dir", + envvar="DBT_PROFILES_DIR", help=f"Which directory to look in for the profiles.yml file. Default = {PurePath.joinpath(Path.home(), '.dbt')}", default=PurePath.joinpath(Path.home(), ".dbt"), type=click.Path( @@ -170,6 +226,7 @@ def convert(self, value, param, ctx): project_dir = click.option( "--project-dir", + envvar=None, help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", default=Path.cwd(), type=click.Path(exists=True), @@ -177,13 +234,15 @@ def convert(self, value, param, ctx): quiet = click.option( "--quiet/--no-quiet", + envvar="DBT_QUIET", help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", default=False, ) record_timing = click.option( - "-r", "--record-timing-info", + "-r", + envvar=None, help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", is_flag=True, default=False, @@ -191,6 +250,7 @@ def convert(self, value, param, ctx): resource_type = click.option( "--resource-type", + envvar=None, help="TODO: No current help text", type=click.Choice( [ @@ -210,16 +270,13 @@ def convert(self, value, param, ctx): default="default", ) -selector = click.option("--selector", help="The selector name to use, as defined in selectors.yml") - -send_anonymous_usage_stats = click.option( - "--anonymous-usage-stats/--no-anonymous-usage-stats", - help="Send anonymous usage stats to dbt Labs.", - default=True, +selector = click.option( + "--selector", envvar=None, help="The selector name to use, as defined in selectors.yml" ) show = click.option( "--show", + envvar=None, help="Show a sample of the loaded data in the terminal", default=False, ) @@ -227,33 +284,50 @@ def convert(self, value, param, ctx): skip_profile_setup = click.option( "--skip-profile-setup", "-s", + envvar=None, help="Skip interative profile setup.", default=False, ) +# TODO: The env var and name (reflected in flags) are corrections! +# The original name was `ARTIFACT_STATE_PATH` and used the env var `DBT_ARTIFACT_STATE_PATH`. +# Both of which break existing naming conventions. +# This will need to be fixed before use in the main codebase and communicated as a change to the community! state = click.option( "--state", + envvar="DBT_STATE", help="If set, use the given directory as the source for json files to compare with this project.", + type=Truthy(), ) static_parser = click.option( - "--static-parser/--no-static-parser", help="Use the static parser.", default=True + "--static-parser/--no-static-parser", + envvar="DBT_STATIC_PARSER", + help="Use the static parser.", + default=True, ) store_failures = click.option( - "--store-failures", help="Store test results (failing rows) in the database", default=False + "--store-failures", + envvar="DBT_STORE_FAILURES", + help="Store test results (failing rows) in the database", + default=False, ) -target = click.option("-t", "--target", help="Which target to load for the given profile") +target = click.option( + "--target", "-t", envvar=None, help="Which target to load for the given profile" +) target_path = click.option( "--target-path", + envvar="DBT_TARGET_PATH", help="Configure the 'target-path'. Only applies this setting for the current run. Overrides the 'DBT_TARGET_PATH' if it is set.", type=click.Path(), ) threads = click.option( "--threads", + envvar=None, help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", default=1, type=click.INT, @@ -261,44 +335,53 @@ def convert(self, value, param, ctx): use_colors = click.option( "--use-colors/--no-use-colors", + envvar="DBT_USE_COLORS", help="Output is colorized by default and may also be set in a profile or at the command line.", default=True, ) use_experimental_parser = click.option( "--use-experimental-parser/--no-use-experimental-parser", + envvar="DBT_USE_EXPERIMENTAL_PARSER", help="Enable experimental parsing features.", default=False, ) vars = click.option( "--vars", + envvar=None, help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", type=YAML(), ) -version = click.option("--version", help="Show version information", is_flag=True, default=False) +version = click.option( + "--version", envvar=None, help="Show version information", is_flag=True, default=False +) version_check = click.option( "--version-check/--no-version-check", + envvar="DBT_VERSION_CHECK", help="Ensure dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version')", default=True, ) warn_error = click.option( "--warn-error/--no-warn-error", + envvar="DBT_WARN_ERROR", help="If dbt would normally warn, instead raise an exception. Examples include --models that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", default=False, ) write_json = click.option( "--write-json/--no-write-json", + envvar="DBT_WRITE_JSON", help="Writing the manifest and run_results.json files to disk", default=True, ) write_manifest = click.option( "--write-manifest/--no-write-manifest", + envvar=None, help="TODO: No help text currently available", default=True, ) diff --git a/core/setup.py b/core/setup.py index 6ac908eaa51..2c254dcf6b0 100644 --- a/core/setup.py +++ b/core/setup.py @@ -43,7 +43,7 @@ include_package_data=True, test_suite="test", entry_points={ - "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:runner"], + "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:cli"], }, install_requires=[ "Jinja2==3.1.2", diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 294961c98e7..ef8bd16a29d 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,25 +1,40 @@ -from dbt.cli.main import cli import click +from dbt.cli.main import cli class TestCLI: def test_commands_have_docstrings(self): def run_test(commands): - for _, command in commands.items(): - if type(command) is click.core.Command: + for command in commands.values(): + if type(command) is click.Command: assert command.__doc__ is not None - if type(command) is click.core.Group: + if type(command) is click.Group: run_test(command.commands) run_test(cli.commands) - def test_params_have_help_texts(self): - def run_test(commands): - for _, command in commands.items(): - if type(command) is click.core.Command: - for param in command.params: - assert param.help is not None - if type(command) is click.core.Group: - run_test(command.commands) + # TODO: This isn't the ideal way to test params as + # they will be tested as many times as they are used as decorators. + # This is inefficent (obvs) + def test_unhidden_params_have_help_texts(self): + def run_test(command): + for param in command.params: + if not param.hidden: + assert param.help is not None + if type(command) is click.Group: + for command in command.commands.values(): + run_test(command) - run_test(cli.commands) + run_test(cli) + + def test_params_follow_naming_convention(self): + def run_test(command): + for param in command.params: + assert param.name.replace("_", "-") in " ".join(param.opts) + if param.envvar is not None: + assert "DBT_" + param.name.upper() == param.envvar + if type(command) is click.Group: + for command in command.commands.values(): + run_test(command) + + run_test(cli) From 1571a43ccc38765d0a73a59e724b827b8d955331 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Wed, 7 Sep 2022 13:27:44 -0500 Subject: [PATCH 10/21] more tests, alpha-order params --- core/dbt/cli/flags.py | 1 - core/dbt/cli/main.py | 195 ++++++++++++++++++++++------------------- core/dbt/cli/params.py | 2 +- tests/unit/test_cli.py | 10 +++ 4 files changed, 118 insertions(+), 90 deletions(-) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 7c305cd77e3..de88113b008 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -33,7 +33,6 @@ def assign_params(ctx): # Hard coded flags # TODO: This is listed as for backwards compat... Still needed? object.__setattr__(self, "STRICT_MODE", False) - # TODO: Appears unused... Still needed? object.__setattr__(self, "WHICH", None) object.__setattr__(self, "MP_CONTEXT", get_context("spawn")) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index a9a261720f2..28117446b52 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -16,10 +16,10 @@ ) @click.pass_context @p.anonymous_usage_stats -@p.version @p.cache_selected_only @p.debug @p.enable_legacy_logger +@p.event_buffer_size @p.fail_fast @p.log_cache_events @p.log_format @@ -28,14 +28,14 @@ @p.print @p.printer_width @p.quiet +@p.record_timing @p.static_parser @p.use_colors @p.use_experimental_parser +@p.version @p.version_check @p.warn_error @p.write_json -@p.event_buffer_size -@p.record_timing def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com @@ -50,6 +50,25 @@ def cli(ctx, **kwargs): # dbt build @cli.command("build") @click.pass_context +@p.defer +@p.exclude +@p.fail_fast +@p.full_refresh +@p.indirect_selection +@p.log_path +@p.models +@p.profile +@p.profiles_dir +@p.project_dir +@p.selector +@p.show +@p.state +@p.store_failures +@p.target +@p.target_path +@p.threads +@p.vars +@p.version_check def build(ctx, **kwargs): """Run all Seeds, Models, Snapshots, and tests in DAG order""" click.echo( @@ -60,9 +79,9 @@ def build(ctx, **kwargs): # dbt clean @cli.command("clean") @click.pass_context -@p.project_dir -@p.profiles_dir @p.profile +@p.profiles_dir +@p.project_dir @p.target @p.vars def clean(ctx, **kwargs): @@ -83,21 +102,21 @@ def docs(ctx, **kwargs): # dbt docs generate @docs.command("generate") @click.pass_context -@p.version_check -@p.project_dir -@p.profiles_dir -@p.profile -@p.target -@p.vars @p.compile_docs @p.defer -@p.threads -@p.target_path +@p.exclude @p.log_path @p.models -@p.exclude +@p.profile +@p.profiles_dir +@p.project_dir @p.selector @p.state +@p.target +@p.target_path +@p.threads +@p.vars +@p.version_check def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" click.echo( @@ -108,13 +127,13 @@ def docs_generate(ctx, **kwargs): # dbt docs serve @docs.command("serve") @click.pass_context -@p.project_dir -@p.profiles_dir +@p.browser +@p.port @p.profile +@p.profiles_dir +@p.project_dir @p.target @p.vars -@p.port -@p.browser def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" click.echo( @@ -125,22 +144,22 @@ def docs_serve(ctx, **kwargs): # dbt compile @cli.command("compile") @click.pass_context -@p.version_check -@p.project_dir -@p.profiles_dir -@p.profile -@p.target -@p.vars -@p.parse_only -@p.threads -@p.target_path +@p.defer +@p.exclude +@p.full_refresh @p.log_path @p.models -@p.exclude +@p.parse_only +@p.profile +@p.profiles_dir +@p.project_dir @p.selector @p.state -@p.defer -@p.full_refresh +@p.target +@p.target_path +@p.threads +@p.vars +@p.version_check def compile(ctx, **kwargs): """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" click.echo( @@ -151,13 +170,13 @@ def compile(ctx, **kwargs): # dbt debug @cli.command("debug") @click.pass_context -@p.version_check -@p.project_dir -@p.profiles_dir +@p.config_dir @p.profile +@p.profiles_dir +@p.project_dir @p.target @p.vars -@p.config_dir +@p.version_check def debug(ctx, **kwargs): """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" click.echo( @@ -186,9 +205,9 @@ def deps(ctx, **kwargs): @p.profile @p.profiles_dir @p.project_dir +@p.skip_profile_setup @p.target @p.vars -@p.skip_profile_setup def init(ctx, **kwargs): """Initialize a new DBT project.""" click.echo( @@ -200,19 +219,19 @@ def init(ctx, **kwargs): # dbt TODO: Figure out aliasing for ls (or just c/p?) @cli.command("list") @click.pass_context +@p.exclude +@p.indirect_selection +@p.models +@p.output +@p.output_keys @p.profile @p.profiles_dir @p.project_dir -@p.target -@p.vars -@p.output -@p.ouptut_keys @p.resource_type -@p.models -@p.indirect_selection -@p.exclude @p.selector @p.state +@p.target +@p.vars def list(ctx, **kwargs): """List the resources in your project""" click.echo( @@ -223,17 +242,17 @@ def list(ctx, **kwargs): # dbt parse @cli.command("parse") @click.pass_context +@p.compile_parse +@p.log_path @p.profile @p.profiles_dir @p.project_dir @p.target -@p.vars -@p.write_manifest -@p.compile_parse -@p.threads @p.target_path -@p.log_path +@p.threads +@p.vars @p.version_check +@p.write_manifest def parse(ctx, **kwargs): """Parses the project and provides information on performance""" click.echo( @@ -244,22 +263,22 @@ def parse(ctx, **kwargs): # dbt run @cli.command("run") @click.pass_context +@p.defer +@p.exclude @p.fail_fast -@p.version_check +@p.full_refresh +@p.log_path +@p.models @p.profile @p.profiles_dir @p.project_dir +@p.selector +@p.state @p.target -@p.vars -@p.log_path @p.target_path @p.threads -@p.models -@p.exclude -@p.selector -@p.state -@p.defer -@p.full_refresh +@p.vars +@p.version_check def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" click.echo( @@ -270,12 +289,12 @@ def run(ctx, **kwargs): # dbt run operation @cli.command("run-operation") @click.pass_context +@p.args @p.profile @p.profiles_dir @p.project_dir @p.target @p.vars -@p.args def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" click.echo( @@ -286,21 +305,21 @@ def run_operation(ctx, **kwargs): # dbt seed @cli.command("seed") @click.pass_context -@p.version_check +@p.exclude +@p.full_refresh +@p.log_path +@p.models @p.profile @p.profiles_dir @p.project_dir +@p.selector +@p.show +@p.state @p.target -@p.vars -@p.full_refresh -@p.log_path @p.target_path @p.threads -@p.models -@p.exclude -@p.selector -@p.state -@p.show +@p.vars +@p.version_check def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" click.echo( @@ -311,17 +330,17 @@ def seed(ctx, **kwargs): # dbt snapshot @cli.command("snapshot") @click.pass_context +@p.defer +@p.exclude +@p.models @p.profile @p.profiles_dir @p.project_dir -@p.target -@p.vars -@p.threads -@p.models -@p.exclude @p.selector @p.state -@p.defer +@p.target +@p.threads +@p.vars def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" click.echo( @@ -339,17 +358,17 @@ def source(ctx, **kwargs): # dbt source freshness @source.command("freshness") @click.pass_context +@p.exclude +@p.models +@p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? @p.profile @p.profiles_dir @p.project_dir -@p.target -@p.vars -@p.threads -@p.models -@p.exclude @p.selector @p.state -@p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? +@p.target +@p.threads +@p.vars def freshness(ctx, **kwargs): """Snapshots the current freshness of the project's sources""" click.echo( @@ -360,23 +379,23 @@ def freshness(ctx, **kwargs): # dbt test @cli.command("test") @click.pass_context +@p.defer +@p.exclude @p.fail_fast -@p.version_check -@p.store_failures +@p.indirect_selection +@p.log_path +@p.models @p.profile @p.profiles_dir @p.project_dir +@p.selector +@p.state +@p.store_failures @p.target -@p.vars -@p.indirect_selection -@p.log_path @p.target_path @p.threads -@p.models -@p.exclude -@p.selector -@p.state -@p.defer +@p.vars +@p.version_check def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" click.echo( diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 76bd4c5e434..2f890409a85 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -151,7 +151,7 @@ default="name", ) -ouptut_keys = click.option( +output_keys = click.option( "--output-keys", envvar=None, help="TODO: No current help text", diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index ef8bd16a29d..1c5aeb50fc1 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,5 +1,8 @@ +import ast +from inspect import getsource import click from dbt.cli.main import cli +from dbt.cli import params class TestCLI: @@ -38,3 +41,10 @@ def run_test(command): run_test(command) run_test(cli) + + def test_params_are_alpha_sorted(self): + root_node = ast.parse(getsource(params)) + param_names = [ + node.targets[0].id for node in ast.walk(root_node) if isinstance(node, ast.Assign) + ] + assert param_names == sorted(param_names) From 562cf41b8f21ffc16feffc9b72849b178ddbb191 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Wed, 7 Sep 2022 15:48:31 -0500 Subject: [PATCH 11/21] nm about that alias... nothing to see here --- core/dbt/cli/flags.py | 4 ++ core/dbt/cli/main.py | 83 +++++++++++++++++------------------------- core/dbt/cli/params.py | 11 +++++- tests/unit/test_cli.py | 10 ++--- 4 files changed, 51 insertions(+), 57 deletions(-) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index de88113b008..acef498a137 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -2,6 +2,7 @@ import os from dataclasses import dataclass from multiprocessing import get_context +from pprint import pformat as pf from click import get_current_context @@ -39,3 +40,6 @@ def assign_params(ctx): # Support console DO NOT TRACK initiave if os.getenv("DO_NOT_TRACK", "").lower() in (1, "t", "true", "y", "yes"): object.__setattr__(self, "ANONYMOUS_USAGE_STATS", False) + + def __str__(self) -> str: + return str(pf(self.__dict__)) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 28117446b52..4a402f97d64 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -28,7 +28,7 @@ @p.print @p.printer_width @p.quiet -@p.record_timing +@p.record_timing_info @p.static_parser @p.use_colors @p.use_experimental_parser @@ -71,9 +71,8 @@ def cli(ctx, **kwargs): @p.version_check def build(ctx, **kwargs): """Run all Seeds, Models, Snapshots, and tests in DAG order""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt clean @@ -87,9 +86,7 @@ def build(ctx, **kwargs): def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" flags = Flags() - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}\n flags {flags}" - ) + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt docs @@ -119,9 +116,8 @@ def docs(ctx, **kwargs): @p.version_check def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt docs serve @@ -136,9 +132,8 @@ def docs_generate(ctx, **kwargs): @p.vars def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt compile @@ -162,9 +157,8 @@ def docs_serve(ctx, **kwargs): @p.version_check def compile(ctx, **kwargs): """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt debug @@ -179,9 +173,8 @@ def compile(ctx, **kwargs): @p.version_check def debug(ctx, **kwargs): """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt deps @@ -194,9 +187,8 @@ def debug(ctx, **kwargs): @p.vars def deps(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt init @@ -210,13 +202,12 @@ def deps(ctx, **kwargs): @p.vars def init(ctx, **kwargs): """Initialize a new DBT project.""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt list -# dbt TODO: Figure out aliasing for ls (or just c/p?) +# TODO: Figure out aliasing for ls (or just c/p?) @cli.command("list") @click.pass_context @p.exclude @@ -234,9 +225,8 @@ def init(ctx, **kwargs): @p.vars def list(ctx, **kwargs): """List the resources in your project""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt parse @@ -255,9 +245,8 @@ def list(ctx, **kwargs): @p.write_manifest def parse(ctx, **kwargs): """Parses the project and provides information on performance""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt run @@ -281,9 +270,8 @@ def parse(ctx, **kwargs): @p.version_check def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt run operation @@ -297,9 +285,8 @@ def run(ctx, **kwargs): @p.vars def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt seed @@ -322,9 +309,8 @@ def run_operation(ctx, **kwargs): @p.version_check def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt snapshot @@ -343,9 +329,8 @@ def seed(ctx, **kwargs): @p.vars def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt source @@ -371,9 +356,8 @@ def source(ctx, **kwargs): @p.vars def freshness(ctx, **kwargs): """Snapshots the current freshness of the project's sources""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") # dbt test @@ -398,6 +382,5 @@ def freshness(ctx, **kwargs): @p.version_check def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" - click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" - ) + flags = Flags() + click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 2f890409a85..cb781a663c9 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -141,7 +141,14 @@ default=None, ) -models = click.option("-m", "-s", envvar=None, help="Specify the nodes to include.", multiple=True) +models = click.option( + "-m", + "-s", + "models", + envvar=None, + help="Specify the nodes to include.", + multiple=True, +) output = click.option( "--output", @@ -239,7 +246,7 @@ default=False, ) -record_timing = click.option( +record_timing_info = click.option( "--record-timing-info", "-r", envvar=None, diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 1c5aeb50fc1..07dda952a9d 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,8 +1,9 @@ import ast from inspect import getsource + import click -from dbt.cli.main import cli from dbt.cli import params +from dbt.cli.main import cli class TestCLI: @@ -30,10 +31,9 @@ def run_test(command): run_test(cli) - def test_params_follow_naming_convention(self): + def test_param_names_match_envvars(self): def run_test(command): for param in command.params: - assert param.name.replace("_", "-") in " ".join(param.opts) if param.envvar is not None: assert "DBT_" + param.name.upper() == param.envvar if type(command) is click.Group: @@ -44,7 +44,7 @@ def run_test(command): def test_params_are_alpha_sorted(self): root_node = ast.parse(getsource(params)) - param_names = [ + param_var_names = [ node.targets[0].id for node in ast.walk(root_node) if isinstance(node, ast.Assign) ] - assert param_names == sorted(param_names) + assert param_var_names == sorted(param_var_names) From 5fe939028ceddc89e7c05df85ab2839caa304cd4 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Wed, 7 Sep 2022 15:58:30 -0500 Subject: [PATCH 12/21] removed un-needed defaults --- core/dbt/cli/params.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index cb781a663c9..e34123dc5b1 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -28,7 +28,6 @@ "--cache-selected-only/--no-cache-selected-only", envvar="DBT_CACHE_SELECTED_ONLY", help="Pre cache database objects relevant to selected resource only.", - default=False, ) compile_docs = click.option( @@ -57,7 +56,6 @@ "-d/ ", envvar="DBT_DEBUG", help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", - default=False, ) # TODO: The env var and name (reflected in flags) are corrections! @@ -75,7 +73,6 @@ "--enable-legacy-logger/--no-enable-legacy-logger", envvar="DBT_ENABLE_LEGACY_LOGGER", hidden=True, - default=False, ) event_buffer_size = click.option( @@ -93,7 +90,6 @@ "-x/ ", envvar="DBT_FAIL_FAST", help="Stop execution on first failure.", - default=False, ) full_refresh = click.option( @@ -243,7 +239,6 @@ "--quiet/--no-quiet", envvar="DBT_QUIET", help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", - default=False, ) record_timing_info = click.option( @@ -351,7 +346,6 @@ "--use-experimental-parser/--no-use-experimental-parser", envvar="DBT_USE_EXPERIMENTAL_PARSER", help="Enable experimental parsing features.", - default=False, ) vars = click.option( @@ -376,7 +370,6 @@ "--warn-error/--no-warn-error", envvar="DBT_WARN_ERROR", help="If dbt would normally warn, instead raise an exception. Examples include --models that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", - default=False, ) write_json = click.option( From baf4e4a6efa27f6a9f23277112845eb386665a04 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Wed, 7 Sep 2022 16:56:25 -0500 Subject: [PATCH 13/21] fixed WHICH flag, updated bad or un-needed defaults --- core/dbt/cli/flags.py | 7 ++++--- core/dbt/cli/params.py | 28 ++++++++++++---------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index acef498a137..1bda236a8b6 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -13,9 +13,10 @@ @dataclass(frozen=True) class Flags: - def __init__(self) -> None: + def __init__(self, ctx=None) -> None: - ctx = get_current_context() + if ctx is None: + ctx = get_current_context() def assign_params(ctx): """Recursively adds all click params to flag object""" @@ -34,7 +35,7 @@ def assign_params(ctx): # Hard coded flags # TODO: This is listed as for backwards compat... Still needed? object.__setattr__(self, "STRICT_MODE", False) - object.__setattr__(self, "WHICH", None) + object.__setattr__(self, "WHICH", ctx.info_name) object.__setattr__(self, "MP_CONTEXT", get_context("spawn")) # Support console DO NOT TRACK initiave diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index e34123dc5b1..d4bf1eca8b9 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -3,6 +3,10 @@ import click from dbt.cli.option_types import YAML, Truthy +# TODO: The name (reflected in flags) is a correction! +# The original name was `SEND_ANONYMOUS_USAGE_STATS` and used an env var called "DBT_SEND_ANONYMOUS_USAGE_STATS" +# Both of which break existing naming conventions (doesn't match param flag). +# This will need to be fixed before use in the main codebase and communicated as a change to the community! anonymous_usage_stats = click.option( "--anonymous-usage-stats/--no-anonymous-usage-stats", envvar="DBT_ANONYMOUS_USAGE_STATS", @@ -66,7 +70,6 @@ "--defer/--no-defer", envvar="DBT_DEFER", help="If set, defer to the state variable for resolving unselected nodes.", - default=True, ) enable_legacy_logger = click.option( @@ -112,7 +115,6 @@ "--log-cache-events/--no-log-cache-events", envvar="DBT_LOG_CACHE_EVENTS", hidden=True, - default=None, ) log_format = click.option( @@ -134,7 +136,6 @@ "--macro-debugging/--no-macro-debugging", envvar="DBT_MACRO_DEBUGGING", hidden=True, - default=None, ) models = click.option( @@ -158,7 +159,7 @@ "--output-keys", envvar=None, help="TODO: No current help text", - default=False, + is_flag=True, ) output_path = click.option( @@ -247,7 +248,6 @@ envvar=None, help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", is_flag=True, - default=False, ) resource_type = click.option( @@ -277,18 +277,11 @@ ) show = click.option( - "--show", - envvar=None, - help="Show a sample of the loaded data in the terminal", - default=False, + "--show", envvar=None, help="Show a sample of the loaded data in the terminal", is_flag=True ) skip_profile_setup = click.option( - "--skip-profile-setup", - "-s", - envvar=None, - help="Skip interative profile setup.", - default=False, + "--skip-profile-setup", "-s", envvar=None, help="Skip interative profile setup.", is_flag=True ) # TODO: The env var and name (reflected in flags) are corrections! @@ -313,7 +306,7 @@ "--store-failures", envvar="DBT_STORE_FAILURES", help="Store test results (failing rows) in the database", - default=False, + is_flag=True, ) target = click.option( @@ -356,7 +349,10 @@ ) version = click.option( - "--version", envvar=None, help="Show version information", is_flag=True, default=False + "--version", + envvar=None, + help="Show version information", + is_flag=True, ) version_check = click.option( From a4efd79df985afec657e9f93fd979ed6167d79f0 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 8 Sep 2022 12:37:11 -0500 Subject: [PATCH 14/21] changelog --- .changes/unreleased/Features-20220908-123650.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Features-20220908-123650.yaml diff --git a/.changes/unreleased/Features-20220908-123650.yaml b/.changes/unreleased/Features-20220908-123650.yaml new file mode 100644 index 00000000000..a68f1d55b23 --- /dev/null +++ b/.changes/unreleased/Features-20220908-123650.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Flags work with new Click CLI +time: 2022-09-08T12:36:50.386978-05:00 +custom: + Author: iknox-fa + Issue: "5529" + PR: "5790" From ee4dd24be4bf661256fa71a95a53aef188ed07ca Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 8 Sep 2022 14:18:28 -0500 Subject: [PATCH 15/21] PR feedback --- core/dbt/cli/params.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index d4bf1eca8b9..e72578b1f23 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -156,10 +156,7 @@ ) output_keys = click.option( - "--output-keys", - envvar=None, - help="TODO: No current help text", - is_flag=True, + "--output-keys", envvar=None, help="TODO: No current help text", type=click.STRING ) output_path = click.option( From 9bc6cc1f01f2795e27dc4628239927e7ee8dd4ee Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 8 Sep 2022 14:26:22 -0500 Subject: [PATCH 16/21] PR feedback --- core/dbt/cli/params.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index e72578b1f23..6b68f46e18d 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -1,7 +1,7 @@ from pathlib import Path, PurePath import click -from dbt.cli.option_types import YAML, Truthy +from dbt.cli.option_types import YAML # TODO: The name (reflected in flags) is a correction! # The original name was `SEND_ANONYMOUS_USAGE_STATS` and used an env var called "DBT_SEND_ANONYMOUS_USAGE_STATS" @@ -289,7 +289,13 @@ "--state", envvar="DBT_STATE", help="If set, use the given directory as the source for json files to compare with this project.", - type=Truthy(), + type=click.Path( + dir_okay=True, + exists=True, + file_okay=False, + readable=True, + resolve_path=True, + ), ) static_parser = click.option( From a73d2740115095ae527de86daf0e15da9d3bd62b Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Fri, 9 Sep 2022 12:03:44 -0500 Subject: [PATCH 17/21] updates for aliasing ls --- core/dbt/cli/main.py | 11 +++++++++++ core/setup.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 4a402f97d64..7c27a2055e7 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,5 +1,6 @@ import inspect # This is temporary for RAT-ing import sys +from copy import copy from pprint import pformat as pf # This is temporary for RAT-ing import click @@ -7,6 +8,16 @@ from dbt.cli.flags import Flags +def cli_runner(): + # Alias "list" to "ls" + ls = copy(cli.commands["list"]) + ls.hidden = True + cli.add_command(ls, "ls") + + # Run the cli + cli() + + # dbt @click.group( context_settings={"help_option_names": ["-h", "--help"]}, diff --git a/core/setup.py b/core/setup.py index 2c254dcf6b0..e78b6a12b45 100644 --- a/core/setup.py +++ b/core/setup.py @@ -43,7 +43,7 @@ include_package_data=True, test_suite="test", entry_points={ - "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:cli"], + "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:cli_runner"], }, install_requires=[ "Jinja2==3.1.2", From 899712eb229aba5aec7931937d4c2049f3efd8d1 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Fri, 9 Sep 2022 12:35:23 -0500 Subject: [PATCH 18/21] made module runnable, removed entrypoint for now --- core/dbt/cli/main.py | 5 +++++ core/setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 7c27a2055e7..a4cb8f15152 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -395,3 +395,8 @@ def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" flags = Flags() click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + + +# Support running as a module +if __name__ == "__main__": + cli_runner() diff --git a/core/setup.py b/core/setup.py index e78b6a12b45..84cc688be28 100644 --- a/core/setup.py +++ b/core/setup.py @@ -43,7 +43,7 @@ include_package_data=True, test_suite="test", entry_points={ - "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:cli_runner"], + "console_scripts": ["dbt = dbt.main:main"], }, install_requires=[ "Jinja2==3.1.2", From 1bee3a8e8c4d730c64412204fe93bd22a8fae2d8 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Fri, 9 Sep 2022 14:14:57 -0500 Subject: [PATCH 19/21] PR feedback --- core/dbt/cli/params.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 6b68f46e18d..68b3b0b8bf0 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -110,11 +110,10 @@ default="eager", ) -# TODO: Should this be exposed in the help text? log_cache_events = click.option( "--log-cache-events/--no-log-cache-events", + help="Enable verbose adapter cache logging.", envvar="DBT_LOG_CACHE_EVENTS", - hidden=True, ) log_format = click.option( From a61fa68c33a33d53d6f567f79c18ab9174c1c50a Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Fri, 9 Sep 2022 14:15:56 -0500 Subject: [PATCH 20/21] PR feedback --- core/dbt/cli/flags.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 1bda236a8b6..3593a69de84 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -33,8 +33,6 @@ def assign_params(ctx): assign_params(ctx) # Hard coded flags - # TODO: This is listed as for backwards compat... Still needed? - object.__setattr__(self, "STRICT_MODE", False) object.__setattr__(self, "WHICH", ctx.info_name) object.__setattr__(self, "MP_CONTEXT", get_context("spawn")) From ce203be7d8f3c680d448fdae0890b3cff56eae0e Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Fri, 9 Sep 2022 14:21:29 -0500 Subject: [PATCH 21/21] removed ls TODO --- core/dbt/cli/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index a4cb8f15152..4e7760b6ce8 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -218,7 +218,6 @@ def init(ctx, **kwargs): # dbt list -# TODO: Figure out aliasing for ls (or just c/p?) @cli.command("list") @click.pass_context @p.exclude