From 2b24a4934ff78c14813d261cb521237cbab1be03 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 14 Aug 2020 13:07:28 -0600 Subject: [PATCH 1/2] defer tests, too --- CHANGELOG.md | 4 +-- core/dbt/main.py | 30 +++++++++++-------- core/dbt/task/seed.py | 4 +++ core/dbt/task/snapshot.py | 4 +++ .../062_defer_state_test/test_defer_state.py | 23 ++++++++++---- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e29e2a78670..0cd6f8d1560 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ ## dbt 0.19.0 (Release TBD) ### Features -- Added macro get_partitions_metadata(table) to return partition metadata for partitioned table [#2596](https://github.com/fishtown-analytics/dbt/pull/2596) +- Added macro get_partitions_metadata(table) to return partition metadata for partitioned table ([#2552](https://github.com/fishtown-analytics/dbt/pull/2552), [#2596](https://github.com/fishtown-analytics/dbt/pull/2596)) +- Added `--defer` flag for `dbt test` as well. ([#2701](https://github.com/fishtown-analytics/dbt/issues/2701), [#2706](https://github.com/fishtown-analytics/dbt/pull/2706)) - Added native python 're' module for regex in jinja templates [#2851](https://github.com/fishtown-analytics/dbt/pull/2851) - Store resolved node names in manifest ([#2647](https://github.com/fishtown-analytics/dbt/issues/2647), [#2837](https://github.com/fishtown-analytics/dbt/pull/2837)) - Save selectors dictionary to manifest, allow descriptions ([#2693](https://github.com/fishtown-analytics/dbt/issues/2693), [#2866](https://github.com/fishtown-analytics/dbt/pull/2866)) @@ -176,7 +177,6 @@ Contributors: - Add relevance criteria to site search ([docs#113](https://github.com/fishtown-analytics/dbt-docs/pull/113)) - Support new selector methods, intersection, and arbitrary parent/child depth in DAG selection syntax ([docs#118](https://github.com/fishtown-analytics/dbt-docs/pull/118)) - Revise anonymous event tracking: simpler URL fuzzing; differentiate between Cloud-hosted and non-Cloud docs ([docs#121](https://github.com/fishtown-analytics/dbt-docs/pull/121)) - Contributors: - [@bbhoss](https://github.com/bbhoss) ([#2677](https://github.com/fishtown-analytics/dbt/pull/2677)) - [@kconvey](https://github.com/kconvey) ([#2694](https://github.com/fishtown-analytics/dbt/pull/2694), [#2709](https://github.com/fishtown-analytics/dbt/pull/2709)), [#2711](https://github.com/fishtown-analytics/dbt/pull/2711)) diff --git a/core/dbt/main.py b/core/dbt/main.py index bfa9840b494..69c521b8057 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -446,6 +446,21 @@ def _build_snapshot_subparser(subparsers, base_subparser): return sub +def _add_defer_argument(*subparsers): + for sub in subparsers: + sub.add_optional_argument_inverse( + '--defer', + enable_help=''' + If set, defer to the state variable for resolving unselected nodes. + ''', + disable_help=''' + If set, do not defer to the state variable for resolving unselected + nodes. + ''', + default=flags.DEFER_MODE, + ) + + def _build_run_subparser(subparsers, base_subparser): run_sub = subparsers.add_parser( 'run', @@ -463,19 +478,6 @@ def _build_run_subparser(subparsers, base_subparser): ''' ) - # this is a "dbt run"-only thing, for now - run_sub.add_optional_argument_inverse( - '--defer', - enable_help=''' - If set, defer to the state variable for resolving unselected nodes. - ''', - disable_help=''' - If set, do not defer to the state variable for resolving unselected - nodes. - ''', - default=flags.DEFER_MODE, - ) - run_sub.set_defaults(cls=run_task.RunTask, which='run', rpc_method='run') return run_sub @@ -1033,6 +1035,8 @@ def parse_args(args, cls=DBTArgumentParser): # list_sub sets up its own arguments. _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub) _add_selection_arguments(snapshot_sub, seed_sub, models_name='select') + # --defer + _add_defer_argument(run_sub, test_sub) # --full-refresh _add_table_mutability_arguments(run_sub, compile_sub) diff --git a/core/dbt/task/seed.py b/core/dbt/task/seed.py index efc060a3321..d2a0b2c4776 100644 --- a/core/dbt/task/seed.py +++ b/core/dbt/task/seed.py @@ -38,6 +38,10 @@ def print_result_line(self, result): class SeedTask(RunTask): + def defer_to_manifest(self, adapter, selected_uids): + # seeds don't defer + return + def raise_on_first_error(self): return False diff --git a/core/dbt/task/snapshot.py b/core/dbt/task/snapshot.py index edc5ba9b277..c247dd77478 100644 --- a/core/dbt/task/snapshot.py +++ b/core/dbt/task/snapshot.py @@ -22,6 +22,10 @@ class SnapshotTask(RunTask): def raise_on_first_error(self): return False + def defer_to_manifest(self, adapter, selected_uids): + # snapshots don't defer + return + def get_node_selector(self): if self.manifest is None or self.graph is None: raise InternalException( diff --git a/test/integration/062_defer_state_test/test_defer_state.py b/test/integration/062_defer_state_test/test_defer_state.py index ebdd2421e0c..8c73b33e36c 100644 --- a/test/integration/062_defer_state_test/test_defer_state.py +++ b/test/integration/062_defer_state_test/test_defer_state.py @@ -27,7 +27,7 @@ def project_config(self): 'config-version': 2, 'seeds': { 'test': { - 'quote_columns': True, + 'quote_columns': False, } } } @@ -55,13 +55,22 @@ def run_and_defer(self): results = self.run_dbt(['run']) assert len(results) == 2 assert not any(r.node.deferred for r in results) + results = self.run_dbt(['test']) + assert len(results) == 2 # copy files over from the happy times when we had a good target self.copy_state() - # no state, still fails + # test tests first, because run will change things + # no state, wrong schema, failure. + self.run_dbt(['test', '--target', 'otherschema'], expect_pass=False) + + # no state, run also fails self.run_dbt(['run', '--target', 'otherschema'], expect_pass=False) + # defer test, it succeeds + results = self.run_dbt(['test', '-m', 'view_model+', '--state', 'state', '--defer', '--target', 'otherschema']) + # with state it should work though results = self.run_dbt(['run', '-m', 'view_model', '--state', 'state', '--defer', '--target', 'otherschema']) assert self.other_schema not in results[0].node.compiled_sql @@ -115,23 +124,25 @@ def run_defer_iff_not_exists(self): assert len(results) == 1 results = self.run_dbt(['run', '--state', 'state', '--defer']) assert len(results) == 2 - + # because the seed now exists in our schema, we shouldn't defer it assert self.other_schema not in results[0].node.compiled_sql assert self.unique_schema() in results[0].node.compiled_sql + # despite deferral, test should use models just created in our schema + results = self.run_dbt(['test', '--state', 'state', '--defer']) + assert self.other_schema not in results[0].node.compiled_sql + assert self.unique_schema() in results[0].node.compiled_sql + @use_profile('postgres') def test_postgres_state_changetarget(self): self.run_and_defer() # these should work without --defer! - self.run_dbt(['test']) self.run_dbt(['snapshot']) # make sure these commands don't work with --defer with pytest.raises(SystemExit): self.run_dbt(['seed', '--defer']) - with pytest.raises(SystemExit): - self.run_dbt(['test', '--defer']) with pytest.raises(SystemExit): self.run_dbt(['snapshot', '--defer']) From 274c3012b028b9a44309da4eb1d90d9d216540c7 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Wed, 16 Dec 2020 14:53:25 -0500 Subject: [PATCH 2/2] Add defer to rpc test method --- core/dbt/contracts/rpc.py | 1 + core/dbt/task/rpc/project_commands.py | 4 ++++ test/rpc/test_test.py | 6 ++++++ test/rpc/util.py | 3 +++ 4 files changed, 14 insertions(+) diff --git a/core/dbt/contracts/rpc.py b/core/dbt/contracts/rpc.py index 2bf7046c382..462ecae99af 100644 --- a/core/dbt/contracts/rpc.py +++ b/core/dbt/contracts/rpc.py @@ -79,6 +79,7 @@ class RPCTestParameters(RPCCompileParameters): data: bool = False schema: bool = False state: Optional[str] = None + defer: Optional[bool] = None @dataclass diff --git a/core/dbt/task/rpc/project_commands.py b/core/dbt/task/rpc/project_commands.py index f9735ec30b9..cc345cfde20 100644 --- a/core/dbt/task/rpc/project_commands.py +++ b/core/dbt/task/rpc/project_commands.py @@ -129,6 +129,10 @@ def set_args(self, params: RPCTestParameters) -> None: self.args.schema = params.schema if params.threads is not None: self.args.threads = params.threads + if params.defer is None: + self.args.defer = flags.DEFER_MODE + else: + self.args.defer = params.defer self.args.state = state_path(params.state) self.set_previous_state() diff --git a/test/rpc/test_test.py b/test/rpc/test_test.py index a928dc49e66..9f938c3c966 100644 --- a/test/rpc/test_test.py +++ b/test/rpc/test_test.py @@ -105,3 +105,9 @@ def test_rpc_test_state( querier.test(state='./state', models=['state:modified']), ) assert len(results['results']) == 0 + + # a better test of defer would require multiple targets + results = querier.async_wait_for_result( + querier.run(state='./state', models=['state:modified'], defer=True) + ) + assert len(results['results']) == 0 diff --git a/test/rpc/util.py b/test/rpc/util.py index 4c3e615a1f0..2492d193ad2 100644 --- a/test/rpc/util.py +++ b/test/rpc/util.py @@ -335,6 +335,7 @@ def test( data: bool = None, schema: bool = None, request_id: int = 1, + defer: Optional[bool] = None, state: Optional[bool] = None, ): params = {} @@ -348,6 +349,8 @@ def test( params['schema'] = schema if threads is not None: params['threads'] = threads + if defer is not None: + params['defer'] = defer if state is not None: params['state'] = state return self.request(