diff --git a/Makefile b/Makefile index 80f707b..62696ff 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Configuration variables -VERSION=1.7.0 +VERSION=1.7.1 PROJ_DIR?=$(shell pwd) VENV_DIR?=${PROJ_DIR}/.bldenv BUILD_DIR=${PROJ_DIR}/build diff --git a/dbt/adapters/oracle/__version__.py b/dbt/adapters/oracle/__version__.py index c0b8b43..39b5e51 100644 --- a/dbt/adapters/oracle/__version__.py +++ b/dbt/adapters/oracle/__version__.py @@ -14,4 +14,4 @@ See the License for the specific language governing permissions and limitations under the License. """ -version = "1.7.3" +version = "1.7.4" diff --git a/dbt/adapters/oracle/impl.py b/dbt/adapters/oracle/impl.py index 4b7c6b8..e0a7f7b 100644 --- a/dbt/adapters/oracle/impl.py +++ b/dbt/adapters/oracle/impl.py @@ -29,9 +29,10 @@ import dbt.exceptions from dbt.adapters.base.relation import BaseRelation, InformationSchema -from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME, ConstraintSupport +from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME, ConstraintSupport, GET_CATALOG_RELATIONS_MACRO_NAME, _expect_row_value from dbt.adapters.sql import SQLAdapter from dbt.adapters.base.meta import available +from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability from dbt.adapters.oracle import OracleAdapterConnectionManager from dbt.adapters.oracle.column import OracleColumn from dbt.adapters.oracle.relation import OracleRelation @@ -95,6 +96,10 @@ class OracleAdapter(SQLAdapter): ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } + _capabilities = CapabilityDict( + {Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full)} + ) + def debug_query(self) -> None: self.execute("select 1 as id from dual") @@ -224,6 +229,69 @@ def _get_one_catalog( results = self._catalog_filter_table(table, manifest) return results + def _get_one_catalog_by_relations( + self, + information_schema: InformationSchema, + relations: List[BaseRelation], + manifest: Manifest, + ) -> agate.Table: + + kwargs = { + "information_schema": information_schema, + "relations": relations, + } + table = self.execute_macro( + GET_CATALOG_RELATIONS_MACRO_NAME, + kwargs=kwargs, + # pass in the full manifest, so we get any local project + # overrides + manifest=manifest, + ) + + # In case database is not defined, we can use the the configured database which we set as part of credentials + for node in chain(manifest.nodes.values(), manifest.sources.values()): + if not node.database or node.database == 'None': + node.database = self.config.credentials.database + + results = self._catalog_filter_table(table, manifest) # type: ignore[arg-type] + return results + + def get_filtered_catalog( + self, manifest: Manifest, relations: Optional[Set[BaseRelation]] = None + ): + catalogs: agate.Table + if ( + relations is None + or len(relations) > 100 + or not self.supports(Capability.SchemaMetadataByRelations) + ): + # Do it the traditional way. We get the full catalog. + catalogs, exceptions = self.get_catalog(manifest) + else: + # Do it the new way. We try to save time by selecting information + # only for the exact set of relations we are interested in. + catalogs, exceptions = self.get_catalog_by_relations(manifest, relations) + + if relations and catalogs: + relation_map = { + ( + r.schema.casefold() if r.schema else None, + r.identifier.casefold() if r.identifier else None, + ) + for r in relations + } + + def in_map(row: agate.Row): + s = _expect_row_value("table_schema", row) + i = _expect_row_value("table_name", row) + s = s.casefold() if s is not None else None + i = i.casefold() if i is not None else None + return (s, i) in relation_map + + catalogs = catalogs.where(in_map) + + return catalogs, exceptions + def list_relations_without_caching( self, schema_relation: BaseRelation, ) -> List[BaseRelation]: diff --git a/dbt/include/oracle/macros/catalog.sql b/dbt/include/oracle/macros/catalog.sql index 9c87def..b68fff5 100644 --- a/dbt/include/oracle/macros/catalog.sql +++ b/dbt/include/oracle/macros/catalog.sql @@ -14,64 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. #} -{% macro oracle__get_catalog(information_schema, schemas) -%} - - {%- call statement('catalog', fetch_result=True) -%} - {# - If the user has multiple databases set and the first one is wrong, this will fail. - But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better. - #} - {% set database = information_schema.database %} - {% if database == 'None' or database is undefined or database is none %} - {% set database = get_database_name() %} - {% endif %} - {{ adapter.verify_database(database) }} - with columns as ( - select - SYS_CONTEXT('userenv', 'DB_NAME') table_catalog, - owner table_schema, - table_name, - column_name, - data_type, - data_type_mod, - decode(data_type_owner, null, TO_CHAR(null), SYS_CONTEXT('userenv', 'DB_NAME')) domain_catalog, - data_type_owner domain_schema, - data_length character_maximum_length, - data_length character_octet_length, - data_length, - data_precision numeric_precision, - data_scale numeric_scale, - nullable is_nullable, - coalesce(column_id, 0) ordinal_position, - default_length, - data_default column_default, - num_distinct, - low_value, - high_value, - density, - num_nulls, - num_buckets, - last_analyzed, - sample_size, - SYS_CONTEXT('userenv', 'DB_NAME') character_set_catalog, - 'SYS' character_set_schema, - SYS_CONTEXT('userenv', 'DB_NAME') collation_catalog, - 'SYS' collation_schema, - character_set_name, - char_col_decl_length, - global_stats, - user_stats, - avg_col_len, - char_length, - char_used, - v80_fmt_image, - data_upgraded, - histogram - from sys.all_tab_columns - ), - tables as - (select SYS_CONTEXT('userenv', 'DB_NAME') table_catalog, +{% macro oracle__get_catalog_tables_sql(information_schema) -%} + select SYS_CONTEXT('userenv', 'DB_NAME') table_catalog, owner table_schema, table_name, case @@ -82,7 +27,7 @@ else 'BASE TABLE' end table_type from sys.all_tables - where upper(table_name) not in (select upper(mview_name) from sys.all_mviews) + where upper(table_name) not in (select upper(mview_name) from sys.all_mviews) union all select SYS_CONTEXT('userenv', 'DB_NAME'), owner, @@ -95,44 +40,146 @@ mview_name, 'MATERIALIZED VIEW' from sys.all_mviews - ) - select - tables.table_catalog as "table_database", - tables.table_schema as "table_schema", - tables.table_name as "table_name", - tables.table_type as "table_type", - all_tab_comments.comments as "table_comment", - columns.column_name as "column_name", - ordinal_position as "column_index", - case - when data_type like '%CHAR%' - then data_type || '(' || cast(char_length as varchar(10)) || ')' - else data_type - end as "column_type", - all_col_comments.comments as "column_comment", - tables.table_schema as "table_owner" - from tables - inner join columns on upper(columns.table_catalog) = upper(tables.table_catalog) - and upper(columns.table_schema) = upper(tables.table_schema) - and upper(columns.table_name) = upper(tables.table_name) - left join all_tab_comments - on upper(all_tab_comments.owner) = upper(tables.table_schema) - and upper(all_tab_comments.table_name) = upper(tables.table_name) - left join all_col_comments - on upper(all_col_comments.owner) = upper(columns.table_schema) - and upper(all_col_comments.table_name) = upper(columns.table_name) - and upper(all_col_comments.column_name) = upper(columns.column_name) - where ( - {%- for schema in schemas -%} - upper(tables.table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%} - ) - order by - tables.table_schema, - tables.table_name, - ordinal_position - {%- endcall -%} +{%- endmacro %} + +{% macro oracle__get_catalog_columns_sql(information_schema) -%} + select + SYS_CONTEXT('userenv', 'DB_NAME') table_catalog, + owner table_schema, + table_name, + column_name, + data_type, + data_type_mod, + decode(data_type_owner, null, TO_CHAR(null), SYS_CONTEXT('userenv', 'DB_NAME')) domain_catalog, + data_type_owner domain_schema, + data_length character_maximum_length, + data_length character_octet_length, + data_length, + data_precision numeric_precision, + data_scale numeric_scale, + nullable is_nullable, + coalesce(column_id, 0) ordinal_position, + default_length, + data_default column_default, + num_distinct, + low_value, + high_value, + density, + num_nulls, + num_buckets, + last_analyzed, + sample_size, + SYS_CONTEXT('userenv', 'DB_NAME') character_set_catalog, + 'SYS' character_set_schema, + SYS_CONTEXT('userenv', 'DB_NAME') collation_catalog, + 'SYS' collation_schema, + character_set_name, + char_col_decl_length, + global_stats, + user_stats, + avg_col_len, + char_length, + char_used, + v80_fmt_image, + data_upgraded, + histogram + from sys.all_tab_columns +{%- endmacro %} + +{% macro oracle__get_catalog_results_sql() -%} + select + tables.table_catalog as "table_database", + tables.table_schema as "table_schema", + tables.table_name as "table_name", + tables.table_type as "table_type", + all_tab_comments.comments as "table_comment", + columns.column_name as "column_name", + ordinal_position as "column_index", + case + when data_type like '%CHAR%' + then data_type || '(' || cast(char_length as varchar(10)) || ')' + else data_type + end as "column_type", + all_col_comments.comments as "column_comment", + tables.table_schema as "table_owner" + from tables + inner join columns on upper(columns.table_catalog) = upper(tables.table_catalog) + and upper(columns.table_schema) = upper(tables.table_schema) + and upper(columns.table_name) = upper(tables.table_name) + left join all_tab_comments + on upper(all_tab_comments.owner) = upper(tables.table_schema) + and upper(all_tab_comments.table_name) = upper(tables.table_name) + left join all_col_comments + on upper(all_col_comments.owner) = upper(columns.table_schema) + and upper(all_col_comments.table_name) = upper(columns.table_name) + and upper(all_col_comments.column_name) = upper(columns.column_name) +{%- endmacro %} + +{% macro oracle__get_catalog_schemas_where_clause_sql(schemas) -%} + where ( + {%- for schema in schemas -%} + upper(tables.table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{%- endmacro %} + +{% macro oracle__get_catalog_relations_where_clause_sql(relations) -%} + where ( + {%- for relation in relations -%} + {% if relation.schema and relation.identifier %} + ( + upper(tables.table_schema) = upper('{{ relation.schema }}') + and upper(tables.table_name) = upper('{{ relation.identifier }}') + ) + {% elif relation.schema %} + ( + upper(tables.table_schema) = upper('{{ relation.schema }}') + ) + {% else %} + {% do exceptions.raise_compiler_error( + '`get_catalog_relations` requires a list of relations, each with a schema' + ) %} + {% endif %} + + {%- if not loop.last %} or {% endif -%} + {%- endfor -%} + ) +{%- endmacro %} + +{% macro oracle__get_catalog(information_schema, schemas) -%} + {% set query %} + with tables as ( + {{ oracle__get_catalog_tables_sql(information_schema) }} + ), + columns as ( + {{ oracle__get_catalog_columns_sql(information_schema) }} + ) + {{ oracle__get_catalog_results_sql() }} + {{ oracle__get_catalog_schemas_where_clause_sql(schemas) }} + order by + tables.table_schema, + tables.table_name, + ordinal_position + {%- endset -%} + {{ return(run_query(query)) }} +{%- endmacro %} + +{% macro oracle__get_catalog_relations(information_schema, relations) -%} + {% set query %} + with tables as ( + {{ oracle__get_catalog_tables_sql(information_schema) }} + ), + columns as ( + {{ oracle__get_catalog_columns_sql(information_schema) }} + ) + {{ oracle__get_catalog_results_sql() }} + {{ oracle__get_catalog_relations_where_clause_sql(relations) }} + order by + tables.table_schema, + tables.table_name, + ordinal_position + {%- endset -%} - {{ return(load_result('catalog').table) }} + {{ return(run_query(query)) }} {%- endmacro %} diff --git a/requirements.txt b/requirements.txt index 09d93d1..db63fd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ dbt-core~=1.7,<1.8 cx_Oracle==8.3.0 -oracledb==1.4.2 +oracledb==2.0.0 diff --git a/setup.cfg b/setup.cfg index baf1753..b7be6e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dbt-oracle -version = 1.7.0 +version = 1.7.1 description = dbt (data build tool) adapter for Oracle Autonomous Database long_description = file: README.md long_description_content_type = text/markdown @@ -34,7 +34,7 @@ include_package_data = True install_requires = dbt-core~=1.7,<1.8 cx_Oracle==8.3.0 - oracledb==1.4.2 + oracledb==2.0.0 test_suite=tests test_requires = dbt-tests-adapter~=1.7,<1.8 diff --git a/setup.py b/setup.py index 84d91db..9e09d27 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ requirements = [ "dbt-core~=1.7,<1.8", "cx_Oracle==8.3.0", - "oracledb==1.4.2" + "oracledb==2.0.0" ] test_requirements = [ @@ -60,7 +60,7 @@ url = 'https://github.com/oracle/dbt-oracle' -VERSION = '1.7.0' +VERSION = '1.7.1' setup( author="Oracle", python_requires='>=3.8',