From 5aa801aaaf2e8f7b162c4d8ebb682bca483c035e Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 20 Jan 2023 15:55:44 -0600 Subject: [PATCH 1/6] init commit for column_types test conversion --- tests/functional/column_types/fixtures.py | 112 ++++++++++++++++++ .../column_types/test_alter_column_types.py | 30 +++++ .../column_types/test_column_types.py | 0 3 files changed, 142 insertions(+) create mode 100644 tests/functional/column_types/fixtures.py create mode 100644 tests/functional/column_types/test_alter_column_types.py create mode 100644 tests/functional/column_types/test_column_types.py diff --git a/tests/functional/column_types/fixtures.py b/tests/functional/column_types/fixtures.py new file mode 100644 index 00000000000..66fed9a06a5 --- /dev/null +++ b/tests/functional/column_types/fixtures.py @@ -0,0 +1,112 @@ +# macros +macro_test_alter_column_type = """ +-- Macro to alter a column type +{% macro test_alter_column_type(model_name, column_name, new_column_type) %} + {% set relation = ref(model_name) %} + {{ alter_column_type(relation, column_name, new_column_type) }} +{% endmacro %} +""" + +macros_sql = """ +{% macro simple_type_check_column(column, check) %} + {% if check == 'string' %} + {{ return(column.is_string()) }} + {% elif check == 'float' %} + {{ return(column.is_float()) }} + {% elif check == 'number' %} + {{ return(column.is_number()) }} + {% elif check == 'numeric' %} + {{ return(column.is_numeric()) }} + {% elif check == 'integer' %} + {{ return(column.is_integer()) }} + {% else %} + {% do exceptions.raise_compiler_error('invalid type check value: ' ~ check) %} + {% endif %} +{% endmacro %} + +{% macro type_check_column(column, type_checks) %} + {% set failures = [] %} + {% for type_check in type_checks %} + {% if type_check.startswith('not ') %} + {% if simple_type_check_column(column, type_check[4:]) %} + {% do log('simple_type_check_column got ', True) %} + {% do failures.append(type_check) %} + {% endif %} + {% else %} + {% if not simple_type_check_column(column, type_check) %} + {% do failures.append(type_check) %} + {% endif %} + {% endif %} + {% endfor %} + {% if (failures | length) > 0 %} + {% do log('column ' ~ column.name ~ ' had failures: ' ~ failures, info=True) %} + {% endif %} + {% do return((failures | length) == 0) %} +{% endmacro %} +{% test is_type(model, column_map) %} + {% if not execute %} + {{ return(None) }} + {% endif %} + {% if not column_map %} + {% do exceptions.raise_compiler_error('test_is_type must have a column name') %} + {% endif %} + {% set columns = adapter.get_columns_in_relation(model) %} + {% if (column_map | length) != (columns | length) %} + {% set column_map_keys = (column_map | list | string) %} + {% set column_names = (columns | map(attribute='name') | list | string) %} + {% do exceptions.raise_compiler_error('did not get all the columns/all columns not specified:\n' ~ column_map_keys ~ '\nvs\n' ~ column_names) %} + {% endif %} + {% set bad_columns = [] %} + {% for column in columns %} + {% set column_key = (column.name | lower) %} + {% if column_key in column_map %} + {% set type_checks = column_map[column_key] %} + {% if not type_checks %} + {% do exceptions.raise_compiler_error('no type checks?') %} + {% endif %} + {% if not type_check_column(column, type_checks) %} + {% do bad_columns.append(column.name) %} + {% endif %} + {% else %} + {% do exceptions.raise_compiler_error('column key ' ~ column_key ~ ' not found in ' ~ (column_map | list | string)) %} + {% endif %} + {% endfor %} + {% do log('bad columns: ' ~ bad_columns, info=True) %} + {% for bad_column in bad_columns %} + select '{{ bad_column }}' as bad_column + {{ 'union all' if not loop.last }} + {% endfor %} + select * from (select 1 limit 0) as nothing +{% endtest %} +""" + +# models/schema + +model_sql = """ +select + 1::smallint as smallint_col, + 2::integer as int_col, + 3::bigint as bigint_col, + 4.0::real as real_col, + 5.0::double precision as double_col, + 6.0::numeric as numeric_col, + '7'::text as text_col, + '8'::varchar(20) as varchar_col +""" + +schema_yml = """ +version: 2 +models: + - name: model + tests: + - is_type: + column_map: + smallint_col: ['integer', 'number'] + int_col: ['integer', 'number'] + bigint_col: ['integer', 'number'] + real_col: ['float', 'number'] + double_col: ['float', 'number'] + numeric_col: ['numeric', 'number'] + text_col: ['string', 'not number'] + varchar_col: ['string', 'not number'] +""" diff --git a/tests/functional/column_types/test_alter_column_types.py b/tests/functional/column_types/test_alter_column_types.py new file mode 100644 index 00000000000..f25d7c6977a --- /dev/null +++ b/tests/functional/column_types/test_alter_column_types.py @@ -0,0 +1,30 @@ +import pytest +from dbt.tests.util import run_dbt +from tests.functional.column_types.fixtures import ( + macro_test_alter_column_type, + model_sql, + schema_yml, +) + + +class TestAlterColumnTypes: + @pytest.fixture(scope="class") + def macros(self): + return { + "test_alter_column_type.sql": macro_test_alter_column_type, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + "schema.yml": schema_yml, + } + + def test_run_alter_and_test(self, project): + kwargs = "model_name, column_name, new_column_type" + results = run_dbt(["run"]) + assert len(results) == 1 + run_dbt(["run-operation", "test_alter_column_type" "--args", kwargs]) + results = run_dbt(["test"]) + assert len(results) == 1 diff --git a/tests/functional/column_types/test_column_types.py b/tests/functional/column_types/test_column_types.py new file mode 100644 index 00000000000..e69de29bb2d From 8d2ed81e7a7728c136d8e00e4f52abb7c92023ec Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 20 Jan 2023 16:32:01 -0600 Subject: [PATCH 2/6] init start of test_column_types.py --- tests/functional/column_types/fixtures.py | 3 +- .../column_types/test_alter_column_types.py | 2 -- .../column_types/test_column_types.py | 33 +++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/tests/functional/column_types/fixtures.py b/tests/functional/column_types/fixtures.py index 66fed9a06a5..97a61c2b6f5 100644 --- a/tests/functional/column_types/fixtures.py +++ b/tests/functional/column_types/fixtures.py @@ -7,7 +7,7 @@ {% endmacro %} """ -macros_sql = """ +macro_test_is_type_sql = """ {% macro simple_type_check_column(column, check) %} {% if check == 'string' %} {{ return(column.is_string()) }} @@ -43,6 +43,7 @@ {% endif %} {% do return((failures | length) == 0) %} {% endmacro %} + {% test is_type(model, column_map) %} {% if not execute %} {{ return(None) }} diff --git a/tests/functional/column_types/test_alter_column_types.py b/tests/functional/column_types/test_alter_column_types.py index f25d7c6977a..ab6291d3ed0 100644 --- a/tests/functional/column_types/test_alter_column_types.py +++ b/tests/functional/column_types/test_alter_column_types.py @@ -2,7 +2,6 @@ from dbt.tests.util import run_dbt from tests.functional.column_types.fixtures import ( macro_test_alter_column_type, - model_sql, schema_yml, ) @@ -17,7 +16,6 @@ def macros(self): @pytest.fixture(scope="class") def models(self): return { - "model.sql": model_sql, "schema.yml": schema_yml, } diff --git a/tests/functional/column_types/test_column_types.py b/tests/functional/column_types/test_column_types.py index e69de29bb2d..93c0fd9c5de 100644 --- a/tests/functional/column_types/test_column_types.py +++ b/tests/functional/column_types/test_column_types.py @@ -0,0 +1,33 @@ +import pytest +from dbt.tests.util import run_dbt +from tests.functional.column_types.fixtures import macro_test_is_type_sql, model_sql, schema_yml + + +class BaseTestColumnTypes: + @pytest.fixture(scope="class") + def macros(self): + return {"test_is_type.sql": macro_test_is_type_sql} + + def run_and_test(self, project): + results = run_dbt(["run"]) + assert len(results) == 1 + results = run_dbt(["test"]) + assert len(results) == 1 + + +class TestColumnTypes(BaseTestColumnTypes): + @pytest.fixture(scope="class") + def models(self): + return {"shcema.yml": schema_yml} + + def test_run_and_test(self, project): + self.run_and_test(project) + + +class TestPosgresColumnTypes(BaseTestColumnTypes): + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": model_sql} + + def test_run_and_test(self, project): + self.run_and_test(project) From 22d8221c7eb15d18b20dc5b374eda6793f26b507 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 20 Jan 2023 16:58:52 -0600 Subject: [PATCH 3/6] pass tes macros into both tests --- tests/functional/column_types/test_alter_column_types.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/column_types/test_alter_column_types.py b/tests/functional/column_types/test_alter_column_types.py index ab6291d3ed0..8d703288228 100644 --- a/tests/functional/column_types/test_alter_column_types.py +++ b/tests/functional/column_types/test_alter_column_types.py @@ -2,6 +2,7 @@ from dbt.tests.util import run_dbt from tests.functional.column_types.fixtures import ( macro_test_alter_column_type, + macro_test_is_type_sql, schema_yml, ) @@ -11,6 +12,7 @@ class TestAlterColumnTypes: def macros(self): return { "test_alter_column_type.sql": macro_test_alter_column_type, + "test_is_type.sql": macro_test_is_type_sql, } @pytest.fixture(scope="class") From a54dd517f97606cdd985f2456c4fb0a95aa87ea2 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Mon, 23 Jan 2023 14:37:11 -0600 Subject: [PATCH 4/6] remove alt tests, remove old tests, push up working conversion --- .../macros/test_alter_column_type.sql | 5 -- .../macros/test_is_type.sql | 72 ------------------- .../056_column_type_tests/pg_models/model.sql | 9 --- .../pg_models/schema.yml | 14 ---- .../test_alter_column_types.py | 13 ---- .../test_column_types.py | 22 ------ .../column_types/test_alter_column_types.py | 30 -------- .../column_types/test_column_types.py | 15 +--- 8 files changed, 3 insertions(+), 177 deletions(-) delete mode 100644 test/integration/056_column_type_tests/macros/test_alter_column_type.sql delete mode 100644 test/integration/056_column_type_tests/macros/test_is_type.sql delete mode 100644 test/integration/056_column_type_tests/pg_models/model.sql delete mode 100644 test/integration/056_column_type_tests/pg_models/schema.yml delete mode 100644 test/integration/056_column_type_tests/test_alter_column_types.py delete mode 100644 test/integration/056_column_type_tests/test_column_types.py delete mode 100644 tests/functional/column_types/test_alter_column_types.py diff --git a/test/integration/056_column_type_tests/macros/test_alter_column_type.sql b/test/integration/056_column_type_tests/macros/test_alter_column_type.sql deleted file mode 100644 index 133d59fada5..00000000000 --- a/test/integration/056_column_type_tests/macros/test_alter_column_type.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Macro to alter a column type -{% macro test_alter_column_type(model_name, column_name, new_column_type) %} - {% set relation = ref(model_name) %} - {{ alter_column_type(relation, column_name, new_column_type) }} -{% endmacro %} diff --git a/test/integration/056_column_type_tests/macros/test_is_type.sql b/test/integration/056_column_type_tests/macros/test_is_type.sql deleted file mode 100644 index 2f1ffde2b1e..00000000000 --- a/test/integration/056_column_type_tests/macros/test_is_type.sql +++ /dev/null @@ -1,72 +0,0 @@ - -{% macro simple_type_check_column(column, check) %} - {% if check == 'string' %} - {{ return(column.is_string()) }} - {% elif check == 'float' %} - {{ return(column.is_float()) }} - {% elif check == 'number' %} - {{ return(column.is_number()) }} - {% elif check == 'numeric' %} - {{ return(column.is_numeric()) }} - {% elif check == 'integer' %} - {{ return(column.is_integer()) }} - {% else %} - {% do exceptions.raise_compiler_error('invalid type check value: ' ~ check) %} - {% endif %} -{% endmacro %} - -{% macro type_check_column(column, type_checks) %} - {% set failures = [] %} - {% for type_check in type_checks %} - {% if type_check.startswith('not ') %} - {% if simple_type_check_column(column, type_check[4:]) %} - {% do log('simple_type_check_column got ', True) %} - {% do failures.append(type_check) %} - {% endif %} - {% else %} - {% if not simple_type_check_column(column, type_check) %} - {% do failures.append(type_check) %} - {% endif %} - {% endif %} - {% endfor %} - {% if (failures | length) > 0 %} - {% do log('column ' ~ column.name ~ ' had failures: ' ~ failures, info=True) %} - {% endif %} - {% do return((failures | length) == 0) %} -{% endmacro %} - -{% test is_type(model, column_map) %} - {% if not execute %} - {{ return(None) }} - {% endif %} - {% if not column_map %} - {% do exceptions.raise_compiler_error('test_is_type must have a column name') %} - {% endif %} - {% set columns = adapter.get_columns_in_relation(model) %} - {% if (column_map | length) != (columns | length) %} - {% set column_map_keys = (column_map | list | string) %} - {% set column_names = (columns | map(attribute='name') | list | string) %} - {% do exceptions.raise_compiler_error('did not get all the columns/all columns not specified:\n' ~ column_map_keys ~ '\nvs\n' ~ column_names) %} - {% endif %} - {% set bad_columns = [] %} - {% for column in columns %} - {% set column_key = (column.name | lower) %} - {% if column_key in column_map %} - {% set type_checks = column_map[column_key] %} - {% if not type_checks %} - {% do exceptions.raise_compiler_error('no type checks?') %} - {% endif %} - {% if not type_check_column(column, type_checks) %} - {% do bad_columns.append(column.name) %} - {% endif %} - {% else %} - {% do exceptions.raise_compiler_error('column key ' ~ column_key ~ ' not found in ' ~ (column_map | list | string)) %} - {% endif %} - {% endfor %} - {% do log('bad columns: ' ~ bad_columns, info=True) %} - {% for bad_column in bad_columns %} - select '{{ bad_column }}' as bad_column - {{ 'union all' if not loop.last }} - {% endfor %} - select * from (select 1 limit 0) as nothing -{% endtest %} diff --git a/test/integration/056_column_type_tests/pg_models/model.sql b/test/integration/056_column_type_tests/pg_models/model.sql deleted file mode 100644 index f1b877225a9..00000000000 --- a/test/integration/056_column_type_tests/pg_models/model.sql +++ /dev/null @@ -1,9 +0,0 @@ -select - 1::smallint as smallint_col, - 2::integer as int_col, - 3::bigint as bigint_col, - 4.0::real as real_col, - 5.0::double precision as double_col, - 6.0::numeric as numeric_col, - '7'::text as text_col, - '8'::varchar(20) as varchar_col diff --git a/test/integration/056_column_type_tests/pg_models/schema.yml b/test/integration/056_column_type_tests/pg_models/schema.yml deleted file mode 100644 index 93e309d1b0b..00000000000 --- a/test/integration/056_column_type_tests/pg_models/schema.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: 2 -models: - - name: model - tests: - - is_type: - column_map: - smallint_col: ['integer', 'number'] - int_col: ['integer', 'number'] - bigint_col: ['integer', 'number'] - real_col: ['float', 'number'] - double_col: ['float', 'number'] - numeric_col: ['numeric', 'number'] - text_col: ['string', 'not number'] - varchar_col: ['string', 'not number'] diff --git a/test/integration/056_column_type_tests/test_alter_column_types.py b/test/integration/056_column_type_tests/test_alter_column_types.py deleted file mode 100644 index e06e1f5697c..00000000000 --- a/test/integration/056_column_type_tests/test_alter_column_types.py +++ /dev/null @@ -1,13 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile -import yaml - - -class TestAlterColumnTypes(DBTIntegrationTest): - @property - def schema(self): - return '056_alter_column_types' - - def run_and_alter_and_test(self, alter_column_type_args): - self.assertEqual(len(self.run_dbt(['run'])), 1) - self.run_dbt(['run-operation', 'test_alter_column_type', '--args', alter_column_type_args]) - self.assertEqual(len(self.run_dbt(['test'])), 1) diff --git a/test/integration/056_column_type_tests/test_column_types.py b/test/integration/056_column_type_tests/test_column_types.py deleted file mode 100644 index 66abbb4c970..00000000000 --- a/test/integration/056_column_type_tests/test_column_types.py +++ /dev/null @@ -1,22 +0,0 @@ -from test.integration.base import DBTIntegrationTest, use_profile - - -class TestColumnTypes(DBTIntegrationTest): - @property - def schema(self): - return '056_column_types' - - def run_and_test(self): - self.assertEqual(len(self.run_dbt(['run'])), 1) - self.assertEqual(len(self.run_dbt(['test'])), 1) - - -class TestPostgresColumnTypes(TestColumnTypes): - @property - def models(self): - return 'pg_models' - - @use_profile('postgres') - def test_postgres_column_types(self): - self.run_and_test() - diff --git a/tests/functional/column_types/test_alter_column_types.py b/tests/functional/column_types/test_alter_column_types.py deleted file mode 100644 index 8d703288228..00000000000 --- a/tests/functional/column_types/test_alter_column_types.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest -from dbt.tests.util import run_dbt -from tests.functional.column_types.fixtures import ( - macro_test_alter_column_type, - macro_test_is_type_sql, - schema_yml, -) - - -class TestAlterColumnTypes: - @pytest.fixture(scope="class") - def macros(self): - return { - "test_alter_column_type.sql": macro_test_alter_column_type, - "test_is_type.sql": macro_test_is_type_sql, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": schema_yml, - } - - def test_run_alter_and_test(self, project): - kwargs = "model_name, column_name, new_column_type" - results = run_dbt(["run"]) - assert len(results) == 1 - run_dbt(["run-operation", "test_alter_column_type" "--args", kwargs]) - results = run_dbt(["test"]) - assert len(results) == 1 diff --git a/tests/functional/column_types/test_column_types.py b/tests/functional/column_types/test_column_types.py index 93c0fd9c5de..08b1ebc4cde 100644 --- a/tests/functional/column_types/test_column_types.py +++ b/tests/functional/column_types/test_column_types.py @@ -8,26 +8,17 @@ class BaseTestColumnTypes: def macros(self): return {"test_is_type.sql": macro_test_is_type_sql} - def run_and_test(self, project): + def run_and_test(self): results = run_dbt(["run"]) assert len(results) == 1 results = run_dbt(["test"]) assert len(results) == 1 -class TestColumnTypes(BaseTestColumnTypes): - @pytest.fixture(scope="class") - def models(self): - return {"shcema.yml": schema_yml} - - def test_run_and_test(self, project): - self.run_and_test(project) - - class TestPosgresColumnTypes(BaseTestColumnTypes): @pytest.fixture(scope="class") def models(self): - return {"model.sql": model_sql} + return {"model.sql": model_sql, "schema.yml": schema_yml} def test_run_and_test(self, project): - self.run_and_test(project) + self.run_and_test() From e63244c539f2cab6c7cbc7ec8768f140cba4a7f3 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Mon, 23 Jan 2023 16:19:49 -0600 Subject: [PATCH 5/6] rename base class, move to adapter zone so adapters can use --- .../dbt/tests/adapter}/column_types/fixtures.py | 0 .../dbt/tests/adapter}/column_types/test_column_types.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename tests/{functional => adapter/dbt/tests/adapter}/column_types/fixtures.py (100%) rename tests/{functional => adapter/dbt/tests/adapter}/column_types/test_column_types.py (75%) diff --git a/tests/functional/column_types/fixtures.py b/tests/adapter/dbt/tests/adapter/column_types/fixtures.py similarity index 100% rename from tests/functional/column_types/fixtures.py rename to tests/adapter/dbt/tests/adapter/column_types/fixtures.py diff --git a/tests/functional/column_types/test_column_types.py b/tests/adapter/dbt/tests/adapter/column_types/test_column_types.py similarity index 75% rename from tests/functional/column_types/test_column_types.py rename to tests/adapter/dbt/tests/adapter/column_types/test_column_types.py index 08b1ebc4cde..a63b939b32b 100644 --- a/tests/functional/column_types/test_column_types.py +++ b/tests/adapter/dbt/tests/adapter/column_types/test_column_types.py @@ -1,9 +1,9 @@ import pytest from dbt.tests.util import run_dbt -from tests.functional.column_types.fixtures import macro_test_is_type_sql, model_sql, schema_yml +from dbt.tests.adapter.column_types.fixtures import macro_test_is_type_sql, model_sql, schema_yml -class BaseTestColumnTypes: +class BaseColumnTypes: @pytest.fixture(scope="class") def macros(self): return {"test_is_type.sql": macro_test_is_type_sql} @@ -15,7 +15,7 @@ def run_and_test(self): assert len(results) == 1 -class TestPosgresColumnTypes(BaseTestColumnTypes): +class TestPosgresColumnTypes(BaseColumnTypes): @pytest.fixture(scope="class") def models(self): return {"model.sql": model_sql, "schema.yml": schema_yml} From d34a82d9bde5a47910dc2036904cf515e5de1ac3 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 24 Jan 2023 14:20:29 -0600 Subject: [PATCH 6/6] typo fix --- .../adapter/dbt/tests/adapter/column_types/test_column_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/adapter/dbt/tests/adapter/column_types/test_column_types.py b/tests/adapter/dbt/tests/adapter/column_types/test_column_types.py index a63b939b32b..cc213d36a4b 100644 --- a/tests/adapter/dbt/tests/adapter/column_types/test_column_types.py +++ b/tests/adapter/dbt/tests/adapter/column_types/test_column_types.py @@ -15,7 +15,7 @@ def run_and_test(self): assert len(results) == 1 -class TestPosgresColumnTypes(BaseColumnTypes): +class TestPostgresColumnTypes(BaseColumnTypes): @pytest.fixture(scope="class") def models(self): return {"model.sql": model_sql, "schema.yml": schema_yml}