diff --git a/CHANGELOG.md b/CHANGELOG.md index f01b0dc67dc..7273701ae06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Added 3 more adapter methods that the new dbt-adapter-test suite can use for testing. ([#2492](https://github.com/fishtown-analytics/dbt/issues/2492), [#2721](https://github.com/fishtown-analytics/dbt/pull/2721)) - It is now an error to attempt installing `dbt` with a Python version less than 3.6. (resolves [#2347](https://github.com/fishtown-analytics/dbt/issues/2347)) - Check for Postgres relation names longer than 63 and throw exception. ([#2197](https://github.com/fishtown-analytics/dbt/issues/2197)) +- If column config says quote, use quoting in SQL for adding a comment. ([#2539](https://github.com/fishtown-analytics/dbt/issues/2539)) ### Fixes diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index 4352755b2da..2b3e6f74ee8 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -57,6 +57,7 @@ class ColumnInfo( description: str = '' meta: Dict[str, Any] = field(default_factory=dict) data_type: Optional[str] = None + quote: Optional[bool] = None tags: List[str] = field(default_factory=list) _extra: Dict[str, Any] = field(default_factory=dict) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 60d9d45f93b..ddffea431ca 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -95,12 +95,18 @@ def add( ): tags: List[str] = [] tags.extend(getattr(column, 'tags', ())) + quote: Optional[bool] + if isinstance(column, UnparsedColumn): + quote = column.quote + else: + quote = None self.column_info[column.name] = ColumnInfo( name=column.name, description=description, data_type=data_type, meta=meta, tags=tags, + quote=quote, _extra=column.extra ) diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index 12e7cca47df..bab1b021375 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -155,6 +155,6 @@ {% for column_name in column_dict %} {% set comment = column_dict[column_name]['description'] %} {% set escaped_comment = postgres_escape_comment(comment) %} - comment on column {{ relation }}.{{ column_name }} is {{ escaped_comment }}; + comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] == True else column_name }} is {{ escaped_comment }}; {% endfor %} {% endmacro %} diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index b7dc93c97c9..a8e274467ff 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -980,6 +980,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': 'The user ID number', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -987,6 +988,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -994,6 +996,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The user's email", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -1001,6 +1004,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The user's IP address", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -1008,6 +1012,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -1048,6 +1053,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': 'The user ID number', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -1055,6 +1061,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -1062,6 +1069,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The user's email", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -1069,6 +1077,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The user's IP address", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -1076,6 +1085,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -1135,6 +1145,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': 'The user ID number', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -1142,6 +1153,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -1149,6 +1161,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The user's email", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -1156,6 +1169,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The user's IP address", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -1163,6 +1177,7 @@ def expected_seeded_manifest(self, model_database=None): 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -1459,6 +1474,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'name': 'first_name', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ct': { @@ -1466,6 +1482,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'name': 'ct', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -1528,6 +1545,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'name': 'first_name', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ct': { @@ -1535,6 +1553,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'name': 'ct', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -1596,6 +1615,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'description': 'The user ID number', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -1603,6 +1623,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -1610,6 +1631,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'description': "The user's email", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -1617,6 +1639,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'description': "The user's IP address", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -1624,6 +1647,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -1679,6 +1703,7 @@ def expected_postgres_references_manifest(self, model_database=None): 'name': 'id', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], } }, @@ -1906,6 +1931,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'email', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -1913,6 +1939,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'first_name', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'id': { @@ -1920,6 +1947,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'id', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -1927,6 +1955,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'ip_address', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -1934,6 +1963,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'updated_at', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -1990,6 +2020,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'email', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -1997,6 +2028,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'first_name', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'id': { @@ -2004,6 +2036,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'id', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -2011,6 +2044,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'ip_address', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -2018,6 +2052,7 @@ def expected_bigquery_complex_manifest(self): 'name': 'updated_at', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -2075,6 +2110,7 @@ def expected_bigquery_complex_manifest(self): 'description': 'The first field', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'field_2': { @@ -2082,6 +2118,7 @@ def expected_bigquery_complex_manifest(self): 'description': 'The second field', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'field_3': { @@ -2089,6 +2126,7 @@ def expected_bigquery_complex_manifest(self): 'description': 'The third field', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'nested_field.field_4': { @@ -2096,6 +2134,7 @@ def expected_bigquery_complex_manifest(self): 'description': 'The first nested field', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'nested_field.field_5': { @@ -2103,6 +2142,7 @@ def expected_bigquery_complex_manifest(self): 'description': 'The second nested field', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -2211,6 +2251,7 @@ def expected_bigquery_complex_manifest(self): 'description': 'The user ID number', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -2218,6 +2259,7 @@ def expected_bigquery_complex_manifest(self): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -2225,6 +2267,7 @@ def expected_bigquery_complex_manifest(self): 'description': "The user's email", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -2232,6 +2275,7 @@ def expected_bigquery_complex_manifest(self): 'description': "The user's IP address", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -2239,6 +2283,7 @@ def expected_bigquery_complex_manifest(self): 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -2360,6 +2405,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': 'The user ID number', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -2367,6 +2413,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -2374,6 +2421,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': "The user's email", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -2381,6 +2429,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': "The user's IP address", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -2388,6 +2437,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -2445,6 +2495,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': 'The user ID number', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -2452,6 +2503,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -2459,6 +2511,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': "The user's email", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -2466,6 +2519,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': "The user's IP address", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -2473,6 +2527,7 @@ def expected_redshift_incremental_view_manifest(self): 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -2617,6 +2672,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'id', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -2624,6 +2680,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'first_name', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -2631,6 +2688,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'email', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -2638,6 +2696,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'ip_address', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -2645,6 +2704,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'updated_at', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], } }, @@ -2700,6 +2760,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'id', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -2707,6 +2768,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'first_name', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -2714,6 +2776,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'email', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -2721,6 +2784,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'ip_address', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -2728,6 +2792,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'updated_at', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], } }, @@ -2781,6 +2846,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'id', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -2788,6 +2854,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'first_name', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -2795,6 +2862,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'email', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -2802,6 +2870,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'ip_address', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -2809,6 +2878,7 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'name': 'updated_at', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], } }, @@ -3112,6 +3182,7 @@ def expected_postgres_references_run_results(self): 'name': 'first_name', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ct': { @@ -3119,6 +3190,7 @@ def expected_postgres_references_run_results(self): 'name': 'ct', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -3198,6 +3270,7 @@ def expected_postgres_references_run_results(self): 'name': 'first_name', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ct': { @@ -3205,6 +3278,7 @@ def expected_postgres_references_run_results(self): 'name': 'ct', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, @@ -3279,6 +3353,7 @@ def expected_postgres_references_run_results(self): 'description': 'The user ID number', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'first_name': { @@ -3286,6 +3361,7 @@ def expected_postgres_references_run_results(self): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'email': { @@ -3293,6 +3369,7 @@ def expected_postgres_references_run_results(self): 'description': "The user's email", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'ip_address': { @@ -3300,6 +3377,7 @@ def expected_postgres_references_run_results(self): 'description': "The user's IP address", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, 'updated_at': { @@ -3307,6 +3385,7 @@ def expected_postgres_references_run_results(self): 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, }, diff --git a/test/integration/035_docs_blocks/test_docs_blocks.py b/test/integration/035_docs_blocks/test_docs_blocks.py index e02e258399f..dacddf394f9 100644 --- a/test/integration/035_docs_blocks/test_docs_blocks.py +++ b/test/integration/035_docs_blocks/test_docs_blocks.py @@ -38,6 +38,7 @@ def test_postgres_valid_doc_ref(self): 'description': 'The user ID number', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, model_data['columns']['id'] @@ -48,6 +49,7 @@ def test_postgres_valid_doc_ref(self): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, model_data['columns']['first_name'] @@ -59,6 +61,7 @@ def test_postgres_valid_doc_ref(self): 'description': "The user's last name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, model_data['columns']['last_name'] @@ -86,6 +89,7 @@ def test_postgres_alternative_docs_path(self): 'description': 'The user ID number with alternative text', 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, model_data['columns']['id'] @@ -96,6 +100,7 @@ def test_postgres_alternative_docs_path(self): 'description': "The user's first name", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, model_data['columns']['first_name'] @@ -107,6 +112,7 @@ def test_postgres_alternative_docs_path(self): 'description': "The user's last name in this other file", 'data_type': None, 'meta': {}, + 'quote': None, 'tags': [], }, model_data['columns']['last_name'] diff --git a/test/integration/060_persist_docs_tests/models/quote_model.sql b/test/integration/060_persist_docs_tests/models/quote_model.sql new file mode 100644 index 00000000000..2255b4bd7f0 --- /dev/null +++ b/test/integration/060_persist_docs_tests/models/quote_model.sql @@ -0,0 +1 @@ +select 1 as {{ adapter.quote("2id") }} diff --git a/test/integration/060_persist_docs_tests/models/schema.yml b/test/integration/060_persist_docs_tests/models/schema.yml index 5a909162456..512e85769f7 100644 --- a/test/integration/060_persist_docs_tests/models/schema.yml +++ b/test/integration/060_persist_docs_tests/models/schema.yml @@ -43,6 +43,12 @@ models: -- /* comment */ Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + - name: quote_model + description: "model to test column quotes and comments" + columns: + - name: 2id + description: "My description" + quote: true seeds: - name: seed diff --git a/test/integration/060_persist_docs_tests/test_persist_docs.py b/test/integration/060_persist_docs_tests/test_persist_docs.py index deea4b2323e..08f696db41d 100644 --- a/test/integration/060_persist_docs_tests/test_persist_docs.py +++ b/test/integration/060_persist_docs_tests/test_persist_docs.py @@ -81,7 +81,7 @@ def run_has_comments_pglike(self): with open('target/catalog.json') as fp: catalog_data = json.load(fp) assert 'nodes' in catalog_data - assert len(catalog_data['nodes']) == 3 + assert len(catalog_data['nodes']) == 4 table_node = catalog_data['nodes']['model.test.table_model'] view_node = self._assert_has_table_comments(table_node) @@ -125,7 +125,7 @@ def test_redshift_late_binding_view(self): with open('target/catalog.json') as fp: catalog_data = json.load(fp) assert 'nodes' in catalog_data - assert len(catalog_data['nodes']) == 3 + assert len(catalog_data['nodes']) == 4 table_node = catalog_data['nodes']['model.test.table_model'] view_node = self._assert_has_table_comments(table_node) @@ -226,7 +226,7 @@ def test_bigquery_persist_docs(self): with open('target/catalog.json') as fp: catalog_data = json.load(fp) assert 'nodes' in catalog_data - assert len(catalog_data['nodes']) == 3 # seed, table, and view model + assert len(catalog_data['nodes']) == 4 # seed, table, and view model for node_id in ['table_model_nested', 'view_model_nested']: # check the descriptions using the api