Merge pull request #126 from oracle/dev/oracledb-2.0

v1.7.1

Makefile

@@ -1,5 +1,5 @@
 # Configuration variables
-VERSION=1.7.0
+VERSION=1.7.1
 PROJ_DIR?=$(shell pwd)
 VENV_DIR?=${PROJ_DIR}/.bldenv
 BUILD_DIR=${PROJ_DIR}/build

dbt/adapters/oracle/__version__.py

@@ -14,4 +14,4 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 """
-version = "1.7.3"
+version = "1.7.4"

dbt/adapters/oracle/impl.py

@@ -29,9 +29,10 @@
 
 import dbt.exceptions
 from dbt.adapters.base.relation import BaseRelation, InformationSchema
-from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME, ConstraintSupport
+from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME, ConstraintSupport, GET_CATALOG_RELATIONS_MACRO_NAME, _expect_row_value
 from dbt.adapters.sql import SQLAdapter
 from dbt.adapters.base.meta import available
+from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability
 from dbt.adapters.oracle import OracleAdapterConnectionManager
 from dbt.adapters.oracle.column import OracleColumn
 from dbt.adapters.oracle.relation import OracleRelation
@@ -95,6 +96,10 @@ class OracleAdapter(SQLAdapter):
         ConstraintType.foreign_key: ConstraintSupport.ENFORCED,
     }
 
+    _capabilities = CapabilityDict(
+        {Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full)}
+    )
+
     def debug_query(self) -> None:
         self.execute("select 1 as id from dual")
 
@@ -224,6 +229,69 @@ def _get_one_catalog(
         results = self._catalog_filter_table(table, manifest)
         return results
 
+    def _get_one_catalog_by_relations(
+        self,
+        information_schema: InformationSchema,
+        relations: List[BaseRelation],
+        manifest: Manifest,
+    ) -> agate.Table:
+
+        kwargs = {
+            "information_schema": information_schema,
+            "relations": relations,
+        }
+        table = self.execute_macro(
+            GET_CATALOG_RELATIONS_MACRO_NAME,
+            kwargs=kwargs,
+            # pass in the full manifest, so we get any local project
+            # overrides
+            manifest=manifest,
+        )
+
+        # In case database is not defined, we can use the the configured database which we set as part of credentials
+        for node in chain(manifest.nodes.values(), manifest.sources.values()):
+            if not node.database or node.database == 'None':
+                node.database = self.config.credentials.database
+
+        results = self._catalog_filter_table(table, manifest)  # type: ignore[arg-type]
+        return results
+
+    def get_filtered_catalog(
+        self, manifest: Manifest, relations: Optional[Set[BaseRelation]] = None
+    ):
+        catalogs: agate.Table
+        if (
+            relations is None
+            or len(relations) > 100
+            or not self.supports(Capability.SchemaMetadataByRelations)
+        ):
+            # Do it the traditional way. We get the full catalog.
+            catalogs, exceptions = self.get_catalog(manifest)
+        else:
+            # Do it the new way. We try to save time by selecting information
+            # only for the exact set of relations we are interested in.
+            catalogs, exceptions = self.get_catalog_by_relations(manifest, relations)
+
+        if relations and catalogs:
+            relation_map = {
+                (
+                    r.schema.casefold() if r.schema else None,
+                    r.identifier.casefold() if r.identifier else None,
+                )
+                for r in relations
+            }
+
+            def in_map(row: agate.Row):
+                s = _expect_row_value("table_schema", row)
+                i = _expect_row_value("table_name", row)
+                s = s.casefold() if s is not None else None
+                i = i.casefold() if i is not None else None
+                return (s, i) in relation_map
+
+            catalogs = catalogs.where(in_map)
+
+        return catalogs, exceptions
+
     def list_relations_without_caching(
             self, schema_relation: BaseRelation,
     ) -> List[BaseRelation]:

dbt/include/oracle/macros/catalog.sql

@@ -14,64 +14,9 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 #}
-{% macro oracle__get_catalog(information_schema, schemas) -%}
-
-  {%- call statement('catalog', fetch_result=True) -%}
-    {#
-      If the user has multiple databases set and the first one is wrong, this will fail.
-      But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.
-    #}
-    {% set database = information_schema.database %}
-    {% if database == 'None' or database is undefined or database is none %}
-      {% set database = get_database_name() %}
-    {% endif %}
-    {{ adapter.verify_database(database) }}
 
-    with columns as (
-            select
-                SYS_CONTEXT('userenv', 'DB_NAME') table_catalog,
-                owner table_schema,
-                table_name,
-                column_name,
-                data_type,
-                data_type_mod,
-                decode(data_type_owner, null, TO_CHAR(null), SYS_CONTEXT('userenv', 'DB_NAME')) domain_catalog,
-                data_type_owner domain_schema,
-                data_length character_maximum_length,
-                data_length character_octet_length,
-                data_length,
-                data_precision numeric_precision,
-                data_scale numeric_scale,
-                nullable is_nullable,
-                coalesce(column_id, 0) ordinal_position,
-                default_length,
-                data_default column_default,
-                num_distinct,
-                low_value,
-                high_value,
-                density,
-                num_nulls,
-                num_buckets,
-                last_analyzed,
-                sample_size,
-                SYS_CONTEXT('userenv', 'DB_NAME') character_set_catalog,
-                'SYS' character_set_schema,
-                SYS_CONTEXT('userenv', 'DB_NAME') collation_catalog,
-                'SYS' collation_schema,
-                character_set_name,
-                char_col_decl_length,
-                global_stats,
-                user_stats,
-                avg_col_len,
-                char_length,
-                char_used,
-                v80_fmt_image,
-                data_upgraded,
-                histogram
-              from sys.all_tab_columns
-          ),
-          tables as
-                (select SYS_CONTEXT('userenv', 'DB_NAME') table_catalog,
+{% macro oracle__get_catalog_tables_sql(information_schema) -%}
+    select SYS_CONTEXT('userenv', 'DB_NAME') table_catalog,
                    owner table_schema,
                    table_name,
                    case
@@ -82,7 +27,7 @@
                      else 'BASE TABLE'
                    end table_type
                  from sys.all_tables
-                 where upper(table_name) not in (select upper(mview_name) from sys.all_mviews) 
+                 where upper(table_name) not in (select upper(mview_name) from sys.all_mviews)
                  union all
                  select SYS_CONTEXT('userenv', 'DB_NAME'),
                    owner,
@@ -95,44 +40,146 @@
                    mview_name,
                    'MATERIALIZED VIEW'
                  from sys.all_mviews
-          )
-          select
-              tables.table_catalog as "table_database",
-              tables.table_schema as "table_schema",
-              tables.table_name as "table_name",
-              tables.table_type as "table_type",
-              all_tab_comments.comments as "table_comment",
-              columns.column_name as "column_name",
-              ordinal_position as "column_index",
-              case
-                when data_type like '%CHAR%'
-                then data_type || '(' || cast(char_length as varchar(10)) || ')'
-                else data_type
-              end as "column_type",
-              all_col_comments.comments as "column_comment",
-              tables.table_schema as "table_owner"
-          from tables
-          inner join columns on upper(columns.table_catalog) = upper(tables.table_catalog)
-            and upper(columns.table_schema) = upper(tables.table_schema)
-            and upper(columns.table_name) = upper(tables.table_name)
-          left join all_tab_comments
-            on upper(all_tab_comments.owner) = upper(tables.table_schema)
-              and upper(all_tab_comments.table_name) = upper(tables.table_name)
-          left join all_col_comments
-            on upper(all_col_comments.owner) = upper(columns.table_schema)
-              and upper(all_col_comments.table_name) = upper(columns.table_name)
-              and upper(all_col_comments.column_name) = upper(columns.column_name)
-          where (
-              {%- for schema in schemas -%}
-                upper(tables.table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
-              {%- endfor -%}
-            )
-          order by
-              tables.table_schema,
-              tables.table_name,
-              ordinal_position
-  {%- endcall -%}
+{%- endmacro %}
+
+{% macro oracle__get_catalog_columns_sql(information_schema) -%}
+    select
+        SYS_CONTEXT('userenv', 'DB_NAME') table_catalog,
+        owner table_schema,
+        table_name,
+        column_name,
+        data_type,
+        data_type_mod,
+        decode(data_type_owner, null, TO_CHAR(null), SYS_CONTEXT('userenv', 'DB_NAME')) domain_catalog,
+        data_type_owner domain_schema,
+        data_length character_maximum_length,
+        data_length character_octet_length,
+        data_length,
+        data_precision numeric_precision,
+        data_scale numeric_scale,
+        nullable is_nullable,
+        coalesce(column_id, 0) ordinal_position,
+        default_length,
+        data_default column_default,
+        num_distinct,
+        low_value,
+        high_value,
+        density,
+        num_nulls,
+        num_buckets,
+        last_analyzed,
+        sample_size,
+        SYS_CONTEXT('userenv', 'DB_NAME') character_set_catalog,
+        'SYS' character_set_schema,
+        SYS_CONTEXT('userenv', 'DB_NAME') collation_catalog,
+        'SYS' collation_schema,
+        character_set_name,
+        char_col_decl_length,
+        global_stats,
+        user_stats,
+        avg_col_len,
+        char_length,
+        char_used,
+        v80_fmt_image,
+        data_upgraded,
+        histogram
+      from sys.all_tab_columns
+{%- endmacro %}
+
+{% macro oracle__get_catalog_results_sql() -%}
+    select
+      tables.table_catalog as "table_database",
+      tables.table_schema as "table_schema",
+      tables.table_name as "table_name",
+      tables.table_type as "table_type",
+      all_tab_comments.comments as "table_comment",
+      columns.column_name as "column_name",
+      ordinal_position as "column_index",
+      case
+        when data_type like '%CHAR%'
+        then data_type || '(' || cast(char_length as varchar(10)) || ')'
+        else data_type
+      end as "column_type",
+      all_col_comments.comments as "column_comment",
+      tables.table_schema as "table_owner"
+  from tables
+  inner join columns on upper(columns.table_catalog) = upper(tables.table_catalog)
+    and upper(columns.table_schema) = upper(tables.table_schema)
+    and upper(columns.table_name) = upper(tables.table_name)
+  left join all_tab_comments
+    on upper(all_tab_comments.owner) = upper(tables.table_schema)
+      and upper(all_tab_comments.table_name) = upper(tables.table_name)
+  left join all_col_comments
+    on upper(all_col_comments.owner) = upper(columns.table_schema)
+      and upper(all_col_comments.table_name) = upper(columns.table_name)
+      and upper(all_col_comments.column_name) = upper(columns.column_name)
+{%- endmacro %}
+
+{% macro oracle__get_catalog_schemas_where_clause_sql(schemas) -%}
+     where (
+      {%- for schema in schemas -%}
+        upper(tables.table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
+      {%- endfor -%}
+        )
+{%- endmacro %}
+
+{% macro oracle__get_catalog_relations_where_clause_sql(relations) -%}
+    where (
+        {%- for relation in relations -%}
+            {% if relation.schema and relation.identifier %}
+                (
+                    upper(tables.table_schema) = upper('{{ relation.schema }}')
+                    and upper(tables.table_name) = upper('{{ relation.identifier }}')
+                )
+            {% elif relation.schema %}
+                (
+                    upper(tables.table_schema) = upper('{{ relation.schema }}')
+                )
+            {% else %}
+                {% do exceptions.raise_compiler_error(
+                    '`get_catalog_relations` requires a list of relations, each with a schema'
+                ) %}
+            {% endif %}
+
+            {%- if not loop.last %} or {% endif -%}
+        {%- endfor -%}
+    )
+{%- endmacro %}
+
+{% macro oracle__get_catalog(information_schema, schemas) -%}
+    {% set query %}
+        with tables as (
+            {{ oracle__get_catalog_tables_sql(information_schema) }}
+        ),
+        columns as (
+            {{ oracle__get_catalog_columns_sql(information_schema) }}
+        )
+        {{ oracle__get_catalog_results_sql() }}
+        {{ oracle__get_catalog_schemas_where_clause_sql(schemas) }}
+        order by
+        tables.table_schema,
+        tables.table_name,
+        ordinal_position
+    {%- endset -%}
+    {{ return(run_query(query)) }}
+{%- endmacro %}
+
+{% macro oracle__get_catalog_relations(information_schema, relations) -%}
+    {% set query %}
+        with tables as (
+            {{ oracle__get_catalog_tables_sql(information_schema) }}
+        ),
+        columns as (
+            {{ oracle__get_catalog_columns_sql(information_schema) }}
+        )
+        {{ oracle__get_catalog_results_sql() }}
+        {{ oracle__get_catalog_relations_where_clause_sql(relations) }}
+        order by
+        tables.table_schema,
+        tables.table_name,
+        ordinal_position
+    {%- endset -%}
 
-  {{ return(load_result('catalog').table) }}
+    {{ return(run_query(query)) }}
 
 {%- endmacro %}

requirements.txt

@@ -1,3 +1,3 @@
 dbt-core~=1.7,<1.8
 cx_Oracle==8.3.0
-oracledb==1.4.2
+oracledb==2.0.0

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dbt-oracle
-version = 1.7.0
+version = 1.7.1
 description = dbt (data build tool) adapter for Oracle Autonomous Database
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -34,7 +34,7 @@ include_package_data = True
 install_requires =
     dbt-core~=1.7,<1.8
     cx_Oracle==8.3.0
-    oracledb==1.4.2
+    oracledb==2.0.0
 test_suite=tests
 test_requires =
     dbt-tests-adapter~=1.7,<1.8