From ee1445c42edef652a75e1f37785b10cc5eb36d42 Mon Sep 17 00:00:00 2001 From: maxim-lixakov Date: Tue, 21 May 2024 17:07:29 +0300 Subject: [PATCH] [DOP-15764] - add jdbc option classes --- docs/changelog/next_release/277.feature.rst | 1 + docs/conf.py | 1 + .../db_connection/clickhouse/execute.rst | 16 ++++----- .../db_connection/clickhouse/read.rst | 9 +++-- .../db_connection/clickhouse/sql.rst | 8 +++-- .../db_connection/greenplum/execute.rst | 16 ++++----- .../db_connection/mssql/execute.rst | 17 ++++------ docs/connection/db_connection/mssql/read.rst | 9 +++-- docs/connection/db_connection/mssql/sql.rst | 10 ++++-- .../db_connection/mysql/execute.rst | 16 ++++----- docs/connection/db_connection/mysql/read.rst | 9 +++-- docs/connection/db_connection/mysql/sql.rst | 8 +++-- .../db_connection/oracle/execute.rst | 16 ++++----- docs/connection/db_connection/oracle/read.rst | 9 +++-- docs/connection/db_connection/oracle/sql.rst | 8 +++-- .../db_connection/postgres/execute.rst | 16 ++++----- .../db_connection/postgres/read.rst | 9 +++-- .../connection/db_connection/postgres/sql.rst | 8 +++-- .../db_connection/teradata/execute.rst | 16 ++++----- .../db_connection/teradata/read.rst | 9 +++-- .../connection/db_connection/teradata/sql.rst | 8 +++-- .../db_connection/clickhouse/connection.py | 13 ++++++++ .../db_connection/clickhouse/options.py | 33 +++++++++++++++++++ .../db_connection/greenplum/connection.py | 11 +++++-- .../db_connection/greenplum/options.py | 17 ++++++++++ .../db_connection/jdbc_connection/options.py | 7 +++- .../db_connection/jdbc_mixin/options.py | 6 +++- .../db_connection/mssql/connection.py | 13 ++++++++ .../connection/db_connection/mssql/options.py | 32 ++++++++++++++++++ .../db_connection/mysql/connection.py | 13 ++++++++ .../connection/db_connection/mysql/options.py | 33 +++++++++++++++++++ .../db_connection/oracle/connection.py | 26 +++++++++++---- .../db_connection/oracle/options.py | 33 +++++++++++++++++++ .../db_connection/postgres/connection.py | 19 +++++++++-- .../db_connection/postgres/options.py | 32 ++++++++++++++++++ .../db_connection/teradata/connection.py | 13 ++++++++ .../db_connection/teradata/options.py | 32 ++++++++++++++++++ 37 files changed, 432 insertions(+), 120 deletions(-) create mode 100644 docs/changelog/next_release/277.feature.rst create mode 100644 onetl/connection/db_connection/clickhouse/options.py create mode 100644 onetl/connection/db_connection/mssql/options.py create mode 100644 onetl/connection/db_connection/mysql/options.py create mode 100644 onetl/connection/db_connection/oracle/options.py create mode 100644 onetl/connection/db_connection/postgres/options.py create mode 100644 onetl/connection/db_connection/teradata/options.py diff --git a/docs/changelog/next_release/277.feature.rst b/docs/changelog/next_release/277.feature.rst new file mode 100644 index 000000000..d9f871c57 --- /dev/null +++ b/docs/changelog/next_release/277.feature.rst @@ -0,0 +1 @@ +Add ``ReadOptions``, ``WriteOptions``, ``SQLOptions``, ``FetchOptions``, ``ExecuteOptions`` option classes for each JDBC connector. diff --git a/docs/conf.py b/docs/conf.py index 9427e1902..eaf02cc9f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,6 +57,7 @@ "sphinxcontrib.towncrier", # provides `towncrier-draft-entries` directive "sphinxcontrib.plantuml", "sphinx.ext.extlinks", + "sphinxcontrib.autodoc_pydantic", "sphinx_favicon", "sphinxcontrib.autodoc_pydantic", ] diff --git a/docs/connection/db_connection/clickhouse/execute.rst b/docs/connection/db_connection/clickhouse/execute.rst index 03e5d5df1..c245bf177 100644 --- a/docs/connection/db_connection/clickhouse/execute.rst +++ b/docs/connection/db_connection/clickhouse/execute.rst @@ -20,7 +20,7 @@ Use ``Clickhouse.fetch`` Use this method to perform some ``SELECT`` query which returns **small number or rows**, like reading Clickhouse config, or reading data from some reference table. Method returns Spark DataFrame. -Method accepts :obj:`FetchOptions `. +Method accepts :obj:`ClickhouseFetchOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -60,7 +60,7 @@ Use ``Clickhouse.execute`` Use this method to execute DDL and DML operations. Each method call runs operation in a separated transaction, and then commits it. -Method accepts :obj:`ExecuteOptions `. +Method accepts :obj:`ClickhouseExecuteOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -111,15 +111,13 @@ So it should **NOT** be used to read large amounts of data. Use :ref:`DBReader < Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_mixin.options +.. currentmodule:: onetl.connection.db_connection.clickhouse.options -.. autopydantic_model:: JDBCFetchOptions +.. autopydantic_model:: ClickhouseFetchOptions + :inherited-members: JDBCFetchOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false -.. autopydantic_model:: JDBCExecuteOptions +.. autopydantic_model:: ClickhouseExecuteOptions + :inherited-members: JDBCExecuteOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/clickhouse/read.rst b/docs/connection/db_connection/clickhouse/read.rst index 33bcff4ce..1eaff785c 100644 --- a/docs/connection/db_connection/clickhouse/read.rst +++ b/docs/connection/db_connection/clickhouse/read.rst @@ -22,7 +22,7 @@ Supported DBReader features * * ✅︎ :ref:`incremental-batch-strategy` * ❌ ``hint`` (is not supported by Clickhouse) * ❌ ``df_schema`` -* ✅︎ ``options`` (see :obj:`JDBCReadOptions `) +* ✅︎ ``options`` (see :obj:`ClickhouseReadOptions `) Examples -------- @@ -85,9 +85,8 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.clickhouse.options -.. autopydantic_model:: JDBCReadOptions +.. autopydantic_model:: ClickhouseReadOptions + :inherited-members: JDBCReadOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/clickhouse/sql.rst b/docs/connection/db_connection/clickhouse/sql.rst index a81eda8b9..2b3895b46 100644 --- a/docs/connection/db_connection/clickhouse/sql.rst +++ b/docs/connection/db_connection/clickhouse/sql.rst @@ -70,6 +70,10 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.clickhouse.options -.. autopydantic_model:: JDBCSQLOptions +.. autopydantic_model:: ClickhouseSQLOptions + :inherited-members: JDBCOptions + :member-order: bysource + :model-show-field-summary: false + :field-show-constraints: false diff --git a/docs/connection/db_connection/greenplum/execute.rst b/docs/connection/db_connection/greenplum/execute.rst index dcc32171b..b765e3fb5 100644 --- a/docs/connection/db_connection/greenplum/execute.rst +++ b/docs/connection/db_connection/greenplum/execute.rst @@ -20,7 +20,7 @@ Use ``Greenplum.fetch`` Use this method to perform some ``SELECT`` query which returns **small number or rows**, like reading Greenplum config, or reading data from some reference table. Method returns Spark DataFrame. -Method accepts :obj:`FetchOptions `. +Method accepts :obj:`GreenplumFetchOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -60,7 +60,7 @@ Use ``Greenplum.execute`` Use this method to execute DDL and DML operations. Each method call runs operation in a separated transaction, and then commits it. -Method accepts :obj:`ExecuteOptions `. +Method accepts :obj:`GreenplumExecuteOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -143,15 +143,13 @@ The only port used while interacting with Greenplum in this case is ``5432`` (Gr Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_mixin.options +.. currentmodule:: onetl.connection.db_connection.greenplum.options -.. autopydantic_model:: JDBCFetchOptions +.. autopydantic_model:: GreenplumFetchOptions + :inherited-members: JDBCFetchOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false -.. autopydantic_model:: JDBCExecuteOptions +.. autopydantic_model:: GreenplumExecuteOptions + :inherited-members: JDBCExecuteOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/mssql/execute.rst b/docs/connection/db_connection/mssql/execute.rst index b8b795a66..c2d15ddf4 100644 --- a/docs/connection/db_connection/mssql/execute.rst +++ b/docs/connection/db_connection/mssql/execute.rst @@ -20,7 +20,7 @@ Use ``MSSQL.fetch`` Use this method to perform some ``SELECT`` query which returns **small number or rows**, like reading MSSQL config, or reading data from some reference table. Method returns Spark DataFrame. -Method accepts :obj:`FetchOptions `. +Method accepts :obj:`MSSQLFetchOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -59,7 +59,7 @@ Use ``MSSQL.execute`` Use this method to execute DDL and DML operations. Each method call runs operation in a separated transaction, and then commits it. -Method accepts :obj:`ExecuteOptions `. +Method accepts :obj:`MSSQLExecuteOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -103,15 +103,12 @@ Examples Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_mixin.options +.. currentmodule:: onetl.connection.db_connection.mssql.options -.. autopydantic_model:: JDBCFetchOptions +.. autopydantic_model:: MSSQLFetchOptions + :inherited-members: JDBCFetchOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false - -.. autopydantic_model:: JDBCExecuteOptions +.. autopydantic_model:: MSSQLExecuteOptions + :inherited-members: JDBCExecuteOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/mssql/read.rst b/docs/connection/db_connection/mssql/read.rst index 0c8599aea..59ca55a29 100644 --- a/docs/connection/db_connection/mssql/read.rst +++ b/docs/connection/db_connection/mssql/read.rst @@ -22,7 +22,7 @@ Supported DBReader features * * ✅︎ :ref:`incremental-batch-strategy` * ❌ ``hint`` (MSSQL does support hints, but DBReader not, at least for now) * ❌ ``df_schema`` -* ✅︎ ``options`` (see :obj:`JDBCReadOptions `) +* ✅︎ ``options`` (see :obj:`MSSQLReadOptions `) Examples -------- @@ -85,9 +85,8 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.mssql.options -.. autopydantic_model:: JDBCReadOptions +.. autopydantic_model:: MSSQLReadOptions + :inherited-members: JDBCReadOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/mssql/sql.rst b/docs/connection/db_connection/mssql/sql.rst index de4f6fe63..095ea2afc 100644 --- a/docs/connection/db_connection/mssql/sql.rst +++ b/docs/connection/db_connection/mssql/sql.rst @@ -43,7 +43,7 @@ Examples WHERE key = 'something' """, - options=MSSQL.QLOptions( + options=MSSQL.SQLOptions( partition_column="id", num_partitions=10, lower_bound=0, @@ -70,6 +70,10 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.mssql.options -.. autopydantic_model:: JDBCSQLOptions +.. autopydantic_model:: MSSQLSQLOptions + :inherited-members: JDBCOptions + :member-order: bysource + :model-show-field-summary: false + :field-show-constraints: false diff --git a/docs/connection/db_connection/mysql/execute.rst b/docs/connection/db_connection/mysql/execute.rst index de1f17002..cd9596fc5 100644 --- a/docs/connection/db_connection/mysql/execute.rst +++ b/docs/connection/db_connection/mysql/execute.rst @@ -20,7 +20,7 @@ Use ``MySQL.fetch`` Use this method to perform some ``SELECT`` query which returns **small number or rows**, like reading MySQL config, or reading data from some reference table. Method returns Spark DataFrame. -Method accepts :obj:`FetchOptions `. +Method accepts :obj:`MySQLFetchOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -60,7 +60,7 @@ Use ``MySQL.execute`` Use this method to execute DDL and DML operations. Each method call runs operation in a separated transaction, and then commits it. -Method accepts :obj:`ExecuteOptions `. +Method accepts :obj:`MySQLExecuteOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -104,15 +104,13 @@ Examples Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_mixin.options +.. currentmodule:: onetl.connection.db_connection.mysql.options -.. autopydantic_model:: JDBCFetchOptions +.. autopydantic_model:: MySQLFetchOptions + :inherited-members: JDBCFetchOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false -.. autopydantic_model:: JDBCExecuteOptions +.. autopydantic_model:: MySQLExecuteOptions + :inherited-members: JDBCExecuteOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/mysql/read.rst b/docs/connection/db_connection/mysql/read.rst index e72da45f1..833d3bde5 100644 --- a/docs/connection/db_connection/mysql/read.rst +++ b/docs/connection/db_connection/mysql/read.rst @@ -22,7 +22,7 @@ Supported DBReader features * * ✅︎ :ref:`incremental-batch-strategy` * ✅︎ ``hint`` (see `official documentation `_) * ❌ ``df_schema`` -* ✅︎ ``options`` (see :obj:`JDBCReadOptions `) +* ✅︎ ``options`` (see :obj:`MySQLReadOptions `) Examples -------- @@ -87,9 +87,8 @@ Especially if there are indexes for columns used in ``where`` clause. Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.mysql.options -.. autopydantic_model:: JDBCReadOptions +.. autopydantic_model:: MySQLReadOptions + :inherited-members: JDBCReadOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/mysql/sql.rst b/docs/connection/db_connection/mysql/sql.rst index 949fb3aa3..dbb1e8d98 100644 --- a/docs/connection/db_connection/mysql/sql.rst +++ b/docs/connection/db_connection/mysql/sql.rst @@ -71,6 +71,10 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.mysql.options -.. autopydantic_model:: JDBCSQLOptions +.. autopydantic_model:: MySQLSQLOptions + :inherited-members: JDBCOptions + :member-order: bysource + :model-show-field-summary: false + :field-show-constraints: false diff --git a/docs/connection/db_connection/oracle/execute.rst b/docs/connection/db_connection/oracle/execute.rst index f43eb54be..cfdbb12f6 100644 --- a/docs/connection/db_connection/oracle/execute.rst +++ b/docs/connection/db_connection/oracle/execute.rst @@ -20,7 +20,7 @@ Use ``Oracle.fetch`` Use this method to execute some ``SELECT`` query which returns **small number or rows**, like reading Oracle config, or reading data from some reference table. Method returns Spark DataFrame. -Method accepts :obj:`FetchOptions `. +Method accepts :obj:`OracleFetchOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -60,7 +60,7 @@ Use ``Oracle.execute`` Use this method to execute DDL and DML operations. Each method call runs operation in a separated transaction, and then commits it. -Method accepts :obj:`ExecuteOptions `. +Method accepts :obj:`OracleExecuteOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -104,15 +104,13 @@ Examples Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_mixin.options +.. currentmodule:: onetl.connection.db_connection.oracle.options -.. autopydantic_model:: JDBCFetchOptions +.. autopydantic_model:: OracleFetchOptions + :inherited-members: JDBCFetchOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false -.. autopydantic_model:: JDBCExecuteOptions +.. autopydantic_model:: OracleExecuteOptions + :inherited-members: JDBCExecuteOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/oracle/read.rst b/docs/connection/db_connection/oracle/read.rst index 6592cfc7b..3c6d5c610 100644 --- a/docs/connection/db_connection/oracle/read.rst +++ b/docs/connection/db_connection/oracle/read.rst @@ -22,7 +22,7 @@ Supported DBReader features * * ✅︎ :ref:`incremental-batch-strategy` * ✅︎ ``hint`` (see `official documentation `_) * ❌ ``df_schema`` -* ✅︎ ``options`` (see :obj:`JDBCReadOptions `) +* ✅︎ ``options`` (see :obj:`OracleReadOptions `) Examples -------- @@ -87,9 +87,8 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.oracle.options -.. autopydantic_model:: JDBCReadOptions +.. autopydantic_model:: OracleReadOptions + :inherited-members: JDBCReadOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/oracle/sql.rst b/docs/connection/db_connection/oracle/sql.rst index 5a94daa42..6908aa3d0 100644 --- a/docs/connection/db_connection/oracle/sql.rst +++ b/docs/connection/db_connection/oracle/sql.rst @@ -71,6 +71,10 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.oracle.options -.. autopydantic_model:: JDBCSQLOptions +.. autopydantic_model:: OracleSQLOptions + :inherited-members: JDBCOptions + :member-order: bysource + :model-show-field-summary: false + :field-show-constraints: false diff --git a/docs/connection/db_connection/postgres/execute.rst b/docs/connection/db_connection/postgres/execute.rst index 753c4f624..f58e58776 100644 --- a/docs/connection/db_connection/postgres/execute.rst +++ b/docs/connection/db_connection/postgres/execute.rst @@ -20,7 +20,7 @@ Use ``Postgres.fetch`` Use this method to execute some ``SELECT`` query which returns **small number or rows**, like reading Postgres config, or reading data from some reference table. Method returns Spark DataFrame. -Method accepts :obj:`FetchOptions `. +Method accepts :obj:`PostgresFetchOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -58,7 +58,7 @@ Use ``Postgres.execute`` Use this method to execute DDL and DML operations. Each method call runs operation in a separated transaction, and then commits it. -Method accepts :obj:`ExecuteOptions `. +Method accepts :obj:`PostgresExecuteOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -102,15 +102,13 @@ Examples Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_mixin.options +.. currentmodule:: onetl.connection.db_connection.postgres.options -.. autopydantic_model:: JDBCFetchOptions +.. autopydantic_model:: PostgresFetchOptions + :inherited-members: JDBCFetchOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false -.. autopydantic_model:: JDBCExecuteOptions +.. autopydantic_model:: PostgresExecuteOptions + :inherited-members: JDBCExecuteOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/postgres/read.rst b/docs/connection/db_connection/postgres/read.rst index 67b5234a2..248f34839 100644 --- a/docs/connection/db_connection/postgres/read.rst +++ b/docs/connection/db_connection/postgres/read.rst @@ -22,7 +22,7 @@ Supported DBReader features * * ✅︎ :ref:`incremental-batch-strategy` * ❌ ``hint`` (is not supported by Postgres) * ❌ ``df_schema`` -* ✅︎ ``options`` (see :obj:`JDBCReadOptions `) +* ✅︎ ``options`` (see :obj:`PostgresReadOptions `) Examples -------- @@ -85,9 +85,8 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.postgres.options -.. autopydantic_model:: JDBCReadOptions +.. autopydantic_model:: PostgresReadOptions + :inherited-members: JDBCReadOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/postgres/sql.rst b/docs/connection/db_connection/postgres/sql.rst index f64cf528e..2e041d98a 100644 --- a/docs/connection/db_connection/postgres/sql.rst +++ b/docs/connection/db_connection/postgres/sql.rst @@ -70,6 +70,10 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.postgres.options -.. autopydantic_model:: JDBCSQLOptions +.. autopydantic_model:: PostgresSQLOptions + :inherited-members: JDBCOptions + :member-order: bysource + :model-show-field-summary: false + :field-show-constraints: false diff --git a/docs/connection/db_connection/teradata/execute.rst b/docs/connection/db_connection/teradata/execute.rst index 3d48d0b9a..9b8d66340 100644 --- a/docs/connection/db_connection/teradata/execute.rst +++ b/docs/connection/db_connection/teradata/execute.rst @@ -20,7 +20,7 @@ Use ``Teradata.fetch`` Use this method to execute some ``SELECT`` query which returns **small number or rows**, like reading Teradata config, or reading data from some reference table. Method returns Spark DataFrame. -Method accepts :obj:`FetchOptions `. +Method accepts :obj:`TeradataFetchOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -55,7 +55,7 @@ Use ``Teradata.execute`` Use this method to execute DDL and DML operations. Each method call runs operation in a separated transaction, and then commits it. -Method accepts :obj:`ExecuteOptions `. +Method accepts :obj:`TeradataExecuteOptions `. Connection opened using this method should be then closed with ``connection.close()`` or ``with connection:``. @@ -101,15 +101,13 @@ Examples Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_mixin.options +.. currentmodule:: onetl.connection.db_connection.teradata.options -.. autopydantic_model:: JDBCFetchOptions +.. autopydantic_model:: TeradataFetchOptions + :inherited-members: JDBCFetchOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false -.. autopydantic_model:: JDBCExecuteOptions +.. autopydantic_model:: TeradataExecuteOptions + :inherited-members: JDBCExecuteOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/teradata/read.rst b/docs/connection/db_connection/teradata/read.rst index f4cf95bfb..13cd5391d 100644 --- a/docs/connection/db_connection/teradata/read.rst +++ b/docs/connection/db_connection/teradata/read.rst @@ -18,7 +18,7 @@ Supported DBReader features * * ✅︎ :ref:`incremental-batch-strategy` * ❌ ``hint`` (is not supported by Teradata) * ❌ ``df_schema`` -* ✅︎ ``options`` (see :obj:`JDBCReadOptions `) +* ✅︎ ``options`` (see :obj:`TeradataReadOptions `) Examples -------- @@ -115,9 +115,8 @@ Prefer using ``partitioning_mode="hash"`` from example above. Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.teradata.options -.. autopydantic_model:: JDBCReadOptions +.. autopydantic_model:: TeradataReadOptions + :inherited-members: JDBCReadOptions :member-order: bysource - :model-show-field-summary: false - :field-show-constraints: false diff --git a/docs/connection/db_connection/teradata/sql.rst b/docs/connection/db_connection/teradata/sql.rst index 98b03e107..5dc21ff55 100644 --- a/docs/connection/db_connection/teradata/sql.rst +++ b/docs/connection/db_connection/teradata/sql.rst @@ -68,6 +68,10 @@ Especially if there are indexes or partitions for columns used in ``where`` clau Options ------- -.. currentmodule:: onetl.connection.db_connection.jdbc_connection.options +.. currentmodule:: onetl.connection.db_connection.teradata.options -.. autopydantic_model:: JDBCSQLOptions +.. autopydantic_model:: TeradataSQLOptions + :inherited-members: JDBCOptions + :member-order: bysource + :model-show-field-summary: false + :field-show-constraints: false diff --git a/onetl/connection/db_connection/clickhouse/connection.py b/onetl/connection/db_connection/clickhouse/connection.py index 89b7ff463..3892c5f94 100644 --- a/onetl/connection/db_connection/clickhouse/connection.py +++ b/onetl/connection/db_connection/clickhouse/connection.py @@ -8,6 +8,13 @@ from onetl._util.classproperty import classproperty from onetl._util.version import Version from onetl.connection.db_connection.clickhouse.dialect import ClickhouseDialect +from onetl.connection.db_connection.clickhouse.options import ( + ClickhouseExecuteOptions, + ClickhouseFetchOptions, + ClickhouseReadOptions, + ClickhouseSQLOptions, + ClickhouseWriteOptions, +) from onetl.connection.db_connection.jdbc_connection import JDBCConnection from onetl.connection.db_connection.jdbc_mixin import JDBCStatementType from onetl.hooks import slot, support_hooks @@ -104,6 +111,12 @@ class Clickhouse(JDBCConnection): Extra = ClickhouseExtra Dialect = ClickhouseDialect + ReadOptions = ClickhouseReadOptions + WriteOptions = ClickhouseWriteOptions + SQLOptions = ClickhouseSQLOptions + FetchOptions = ClickhouseFetchOptions + ExecuteOptions = ClickhouseExecuteOptions + DRIVER: ClassVar[str] = "com.clickhouse.jdbc.ClickHouseDriver" @slot diff --git a/onetl/connection/db_connection/clickhouse/options.py b/onetl/connection/db_connection/clickhouse/options.py new file mode 100644 index 000000000..54b4558fb --- /dev/null +++ b/onetl/connection/db_connection/clickhouse/options.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2021-2024 MTS (Mobile Telesystems) +# SPDX-License-Identifier: Apache-2.0 + + +from onetl.connection.db_connection.jdbc_connection.options import ( + JDBCReadOptions, + JDBCSQLOptions, + JDBCWriteOptions, +) +from onetl.connection.db_connection.jdbc_mixin.options import ( + JDBCExecuteOptions, + JDBCFetchOptions, +) + + +class ClickhouseReadOptions(JDBCReadOptions): + pass + + +class ClickhouseWriteOptions(JDBCWriteOptions): + pass + + +class ClickhouseSQLOptions(JDBCSQLOptions): + pass + + +class ClickhouseFetchOptions(JDBCFetchOptions): + pass + + +class ClickhouseExecuteOptions(JDBCExecuteOptions): + pass diff --git a/onetl/connection/db_connection/greenplum/connection.py b/onetl/connection/db_connection/greenplum/connection.py index c9730c44c..77319b3b0 100644 --- a/onetl/connection/db_connection/greenplum/connection.py +++ b/onetl/connection/db_connection/greenplum/connection.py @@ -26,7 +26,10 @@ ) from onetl.connection.db_connection.greenplum.dialect import GreenplumDialect from onetl.connection.db_connection.greenplum.options import ( + GreenplumExecuteOptions, + GreenplumFetchOptions, GreenplumReadOptions, + GreenplumSQLOptions, GreenplumTableExistBehavior, GreenplumWriteOptions, ) @@ -154,10 +157,14 @@ class Greenplum(JDBCMixin, DBConnection): port: int = 5432 extra: GreenplumExtra = GreenplumExtra() - Extra = GreenplumExtra - Dialect = GreenplumDialect ReadOptions = GreenplumReadOptions WriteOptions = GreenplumWriteOptions + SQLOptions = GreenplumSQLOptions + FetchOptions = GreenplumFetchOptions + ExecuteOptions = GreenplumExecuteOptions + + Extra = GreenplumExtra + Dialect = GreenplumDialect DRIVER: ClassVar[str] = "org.postgresql.Driver" CONNECTIONS_WARNING_LIMIT: ClassVar[int] = 31 diff --git a/onetl/connection/db_connection/greenplum/options.py b/onetl/connection/db_connection/greenplum/options.py index e100e35f1..65d275af3 100644 --- a/onetl/connection/db_connection/greenplum/options.py +++ b/onetl/connection/db_connection/greenplum/options.py @@ -11,7 +11,12 @@ except (ImportError, AttributeError): from pydantic import Field, root_validator # type: ignore[no-redef, assignment] +from onetl.connection.db_connection.jdbc_connection.options import JDBCSQLOptions from onetl.connection.db_connection.jdbc_mixin import JDBCOptions +from onetl.connection.db_connection.jdbc_mixin.options import ( + JDBCExecuteOptions, + JDBCFetchOptions, +) # options from which are populated by Greenplum class methods GENERIC_PROHIBITED_OPTIONS = frozenset( @@ -311,3 +316,15 @@ def _mode_is_deprecated(cls, values): stacklevel=3, ) return values + + +class GreenplumSQLOptions(JDBCSQLOptions): + pass + + +class GreenplumFetchOptions(JDBCFetchOptions): + pass + + +class GreenplumExecuteOptions(JDBCExecuteOptions): + pass diff --git a/onetl/connection/db_connection/jdbc_connection/options.py b/onetl/connection/db_connection/jdbc_connection/options.py index 433eaa9fe..f3da923b3 100644 --- a/onetl/connection/db_connection/jdbc_connection/options.py +++ b/onetl/connection/db_connection/jdbc_connection/options.py @@ -4,7 +4,7 @@ import warnings from enum import Enum -from typing import Optional +from typing import Optional, TypeVar try: from pydantic.v1 import Field, PositiveInt, root_validator @@ -625,3 +625,8 @@ def _check_partition_fields(cls, values): class JDBCLegacyOptions(JDBCReadOptions, JDBCWriteOptions): class Config: prohibited_options = JDBCOptions.Config.prohibited_options + + +JDBCReadOptionsType = TypeVar("JDBCReadOptionsType", bound=JDBCReadOptions) +JDBCWriteOptionsType = TypeVar("JDBCWriteOptionsType", bound=JDBCWriteOptions) +JDBCSQLOptionsType = TypeVar("JDBCSQLOptionsType", bound=JDBCSQLOptions) diff --git a/onetl/connection/db_connection/jdbc_mixin/options.py b/onetl/connection/db_connection/jdbc_mixin/options.py index 2fe94c8bb..05dd172d1 100644 --- a/onetl/connection/db_connection/jdbc_mixin/options.py +++ b/onetl/connection/db_connection/jdbc_mixin/options.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from typing import Optional +from typing import Optional, TypeVar from typing_extensions import deprecated @@ -123,3 +123,7 @@ class Config: Default value depends on driver. For example, Oracle has default ``fetchsize=10``. """ + + +JDBCFetchOptionsType = TypeVar("JDBCFetchOptionsType", bound=JDBCFetchOptions) +JDBCExecuteOptionsType = TypeVar("JDBCExecuteOptionsType", bound=JDBCExecuteOptions) diff --git a/onetl/connection/db_connection/mssql/connection.py b/onetl/connection/db_connection/mssql/connection.py index 48143191d..04add7157 100644 --- a/onetl/connection/db_connection/mssql/connection.py +++ b/onetl/connection/db_connection/mssql/connection.py @@ -9,6 +9,13 @@ from onetl._util.version import Version from onetl.connection.db_connection.jdbc_connection import JDBCConnection from onetl.connection.db_connection.mssql.dialect import MSSQLDialect +from onetl.connection.db_connection.mssql.options import ( + MSSQLExecuteOptions, + MSSQLFetchOptions, + MSSQLReadOptions, + MSSQLSQLOptions, + MSSQLWriteOptions, +) from onetl.hooks import slot, support_hooks from onetl.impl import GenericOptions @@ -139,6 +146,12 @@ class MSSQL(JDBCConnection): port: int = 1433 extra: MSSQLExtra = MSSQLExtra() + ReadOptions = MSSQLReadOptions + WriteOptions = MSSQLWriteOptions + SQLOptions = MSSQLSQLOptions + FetchOptions = MSSQLFetchOptions + ExecuteOptions = MSSQLExecuteOptions + Extra = MSSQLExtra Dialect = MSSQLDialect diff --git a/onetl/connection/db_connection/mssql/options.py b/onetl/connection/db_connection/mssql/options.py new file mode 100644 index 000000000..5e5a9d11a --- /dev/null +++ b/onetl/connection/db_connection/mssql/options.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2021-2024 MTS (Mobile Telesystems) +# SPDX-License-Identifier: Apache-2.0 + +from onetl.connection.db_connection.jdbc_connection.options import ( + JDBCReadOptions, + JDBCSQLOptions, + JDBCWriteOptions, +) +from onetl.connection.db_connection.jdbc_mixin.options import ( + JDBCExecuteOptions, + JDBCFetchOptions, +) + + +class MSSQLReadOptions(JDBCReadOptions): + pass + + +class MSSQLWriteOptions(JDBCWriteOptions): + pass + + +class MSSQLSQLOptions(JDBCSQLOptions): + pass + + +class MSSQLFetchOptions(JDBCFetchOptions): + pass + + +class MSSQLExecuteOptions(JDBCExecuteOptions): + pass diff --git a/onetl/connection/db_connection/mysql/connection.py b/onetl/connection/db_connection/mysql/connection.py index a26f8f385..6774870b9 100644 --- a/onetl/connection/db_connection/mysql/connection.py +++ b/onetl/connection/db_connection/mysql/connection.py @@ -9,6 +9,13 @@ from onetl._util.version import Version from onetl.connection.db_connection.jdbc_connection import JDBCConnection from onetl.connection.db_connection.mysql.dialect import MySQLDialect +from onetl.connection.db_connection.mysql.options import ( + MySQLExecuteOptions, + MySQLFetchOptions, + MySQLReadOptions, + MySQLSQLOptions, + MySQLWriteOptions, +) from onetl.hooks import slot, support_hooks from onetl.impl.generic_options import GenericOptions @@ -98,6 +105,12 @@ class MySQL(JDBCConnection): database: Optional[str] = None extra: MySQLExtra = MySQLExtra() + ReadOptions = MySQLReadOptions + WriteOptions = MySQLWriteOptions + SQLOptions = MySQLSQLOptions + FetchOptions = MySQLFetchOptions + ExecuteOptions = MySQLExecuteOptions + Extra = MySQLExtra Dialect = MySQLDialect diff --git a/onetl/connection/db_connection/mysql/options.py b/onetl/connection/db_connection/mysql/options.py new file mode 100644 index 000000000..b2bb2b9d3 --- /dev/null +++ b/onetl/connection/db_connection/mysql/options.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2021-2024 MTS (Mobile Telesystems) +# SPDX-License-Identifier: Apache-2.0 + + +from onetl.connection.db_connection.jdbc_connection.options import ( + JDBCReadOptions, + JDBCSQLOptions, + JDBCWriteOptions, +) +from onetl.connection.db_connection.jdbc_mixin.options import ( + JDBCExecuteOptions, + JDBCFetchOptions, +) + + +class MySQLReadOptions(JDBCReadOptions): + pass + + +class MySQLWriteOptions(JDBCWriteOptions): + pass + + +class MySQLSQLOptions(JDBCSQLOptions): + pass + + +class MySQLFetchOptions(JDBCFetchOptions): + pass + + +class MySQLExecuteOptions(JDBCExecuteOptions): + pass diff --git a/onetl/connection/db_connection/oracle/connection.py b/onetl/connection/db_connection/oracle/connection.py index c21618381..6896aeded 100644 --- a/onetl/connection/db_connection/oracle/connection.py +++ b/onetl/connection/db_connection/oracle/connection.py @@ -21,12 +21,20 @@ from onetl._util.classproperty import classproperty from onetl._util.version import Version from onetl.connection.db_connection.jdbc_connection import JDBCConnection -from onetl.connection.db_connection.jdbc_connection.options import JDBCReadOptions +from onetl.connection.db_connection.jdbc_connection.options import JDBCReadOptionsType from onetl.connection.db_connection.jdbc_mixin.options import ( - JDBCExecuteOptions, + JDBCExecuteOptionsType, + JDBCFetchOptionsType, JDBCOptions, ) from onetl.connection.db_connection.oracle.dialect import OracleDialect +from onetl.connection.db_connection.oracle.options import ( + OracleExecuteOptions, + OracleFetchOptions, + OracleReadOptions, + OracleSQLOptions, + OracleWriteOptions, +) from onetl.hooks import slot, support_hooks from onetl.hwm import Window from onetl.impl import GenericOptions @@ -172,6 +180,12 @@ class Oracle(JDBCConnection): service_name: Optional[str] = None extra: OracleExtra = OracleExtra() + ReadOptions = OracleReadOptions + WriteOptions = OracleWriteOptions + SQLOptions = OracleSQLOptions + FetchOptions = OracleFetchOptions + ExecuteOptions = OracleExecuteOptions + Extra = OracleExtra Dialect = OracleDialect @@ -254,7 +268,7 @@ def get_min_max_values( window: Window, hint: Any | None = None, where: Any | None = None, - options: JDBCReadOptions | None = None, + options: JDBCReadOptionsType | None = None, ) -> tuple[Any, Any]: min_value, max_value = super().get_min_max_values( source=source, @@ -275,7 +289,7 @@ def get_min_max_values( def execute( self, statement: str, - options: JDBCOptions | JDBCExecuteOptions | dict | None = None, # noqa: WPS437 + options: JDBCOptions | JDBCExecuteOptionsType | dict | None = None, # noqa: WPS437 ) -> DataFrame | None: statement = clear_statement(statement) @@ -339,7 +353,7 @@ def _get_compile_errors( type_name: str, schema: str, object_name: str, - options: JDBCOptions | JDBCExecuteOptions, + options: JDBCExecuteOptionsType | JDBCFetchOptionsType, ) -> list[tuple[ErrorPosition, str]]: """ Get compile errors for the object. @@ -409,7 +423,7 @@ def _build_error_message(self, aggregated_errors: OrderedDict[ErrorPosition, str def _handle_compile_errors( self, statement: str, - options: JDBCExecuteOptions, + options: JDBCExecuteOptionsType, ) -> None: """ Oracle does not return compilation errors immediately. diff --git a/onetl/connection/db_connection/oracle/options.py b/onetl/connection/db_connection/oracle/options.py new file mode 100644 index 000000000..2e0b4f9c0 --- /dev/null +++ b/onetl/connection/db_connection/oracle/options.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2021-2024 MTS (Mobile Telesystems) +# SPDX-License-Identifier: Apache-2.0 + + +from onetl.connection.db_connection.jdbc_connection.options import ( + JDBCReadOptions, + JDBCSQLOptions, + JDBCWriteOptions, +) +from onetl.connection.db_connection.jdbc_mixin.options import ( + JDBCExecuteOptions, + JDBCFetchOptions, +) + + +class OracleReadOptions(JDBCReadOptions): + pass + + +class OracleWriteOptions(JDBCWriteOptions): + pass + + +class OracleSQLOptions(JDBCSQLOptions): + pass + + +class OracleFetchOptions(JDBCFetchOptions): + pass + + +class OracleExecuteOptions(JDBCExecuteOptions): + pass diff --git a/onetl/connection/db_connection/postgres/connection.py b/onetl/connection/db_connection/postgres/connection.py index 2b67d43ec..92f53ec54 100644 --- a/onetl/connection/db_connection/postgres/connection.py +++ b/onetl/connection/db_connection/postgres/connection.py @@ -9,11 +9,18 @@ from onetl._util.version import Version from onetl.connection.db_connection.jdbc_connection import JDBCConnection from onetl.connection.db_connection.jdbc_mixin.options import ( - JDBCExecuteOptions, - JDBCFetchOptions, + JDBCExecuteOptionsType, + JDBCFetchOptionsType, JDBCOptions, ) from onetl.connection.db_connection.postgres.dialect import PostgresDialect +from onetl.connection.db_connection.postgres.options import ( + PostgresExecuteOptions, + PostgresFetchOptions, + PostgresReadOptions, + PostgresSQLOptions, + PostgresWriteOptions, +) from onetl.hooks import slot, support_hooks from onetl.impl import GenericOptions @@ -107,6 +114,12 @@ class Postgres(JDBCConnection): port: int = 5432 extra: PostgresExtra = PostgresExtra() + ReadOptions = PostgresReadOptions + WriteOptions = PostgresWriteOptions + SQLOptions = PostgresSQLOptions + FetchOptions = PostgresFetchOptions + ExecuteOptions = PostgresExecuteOptions + Extra = PostgresExtra Dialect = PostgresDialect @@ -165,7 +178,7 @@ def instance_url(self) -> str: def _options_to_connection_properties( self, - options: JDBCOptions | JDBCFetchOptions | JDBCExecuteOptions, + options: JDBCOptions | JDBCFetchOptionsType | JDBCExecuteOptionsType, ): # noqa: WPS437 # See https://github.com/pgjdbc/pgjdbc/pull/1252 # Since 42.2.9 Postgres JDBC Driver added new option readOnlyMode=transaction diff --git a/onetl/connection/db_connection/postgres/options.py b/onetl/connection/db_connection/postgres/options.py new file mode 100644 index 000000000..4f7aecd1d --- /dev/null +++ b/onetl/connection/db_connection/postgres/options.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2021-2024 MTS (Mobile Telesystems) +# SPDX-License-Identifier: Apache-2.0 + +from onetl.connection.db_connection.jdbc_connection.options import ( + JDBCReadOptions, + JDBCSQLOptions, + JDBCWriteOptions, +) +from onetl.connection.db_connection.jdbc_mixin.options import ( + JDBCExecuteOptions, + JDBCFetchOptions, +) + + +class PostgresReadOptions(JDBCReadOptions): + pass + + +class PostgresWriteOptions(JDBCWriteOptions): + pass + + +class PostgresSQLOptions(JDBCSQLOptions): + pass + + +class PostgresFetchOptions(JDBCFetchOptions): + pass + + +class PostgresExecuteOptions(JDBCExecuteOptions): + pass diff --git a/onetl/connection/db_connection/teradata/connection.py b/onetl/connection/db_connection/teradata/connection.py index cf135009d..3f7f884e3 100644 --- a/onetl/connection/db_connection/teradata/connection.py +++ b/onetl/connection/db_connection/teradata/connection.py @@ -10,6 +10,13 @@ from onetl._util.version import Version from onetl.connection.db_connection.jdbc_connection import JDBCConnection from onetl.connection.db_connection.teradata.dialect import TeradataDialect +from onetl.connection.db_connection.teradata.options import ( + TeradataExecuteOptions, + TeradataFetchOptions, + TeradataReadOptions, + TeradataSQLOptions, + TeradataWriteOptions, +) from onetl.hooks import slot from onetl.impl import GenericOptions @@ -118,6 +125,12 @@ class Teradata(JDBCConnection): database: Optional[str] = None extra: TeradataExtra = TeradataExtra() + ReadOptions = TeradataReadOptions + WriteOptions = TeradataWriteOptions + SQLOptions = TeradataSQLOptions + FetchOptions = TeradataFetchOptions + ExecuteOptions = TeradataExecuteOptions + Extra = TeradataExtra Dialect = TeradataDialect diff --git a/onetl/connection/db_connection/teradata/options.py b/onetl/connection/db_connection/teradata/options.py new file mode 100644 index 000000000..c71592c47 --- /dev/null +++ b/onetl/connection/db_connection/teradata/options.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: 2021-2024 MTS (Mobile Telesystems) +# SPDX-License-Identifier: Apache-2.0 + +from onetl.connection.db_connection.jdbc_connection.options import ( + JDBCReadOptions, + JDBCSQLOptions, + JDBCWriteOptions, +) +from onetl.connection.db_connection.jdbc_mixin.options import ( + JDBCExecuteOptions, + JDBCFetchOptions, +) + + +class TeradataReadOptions(JDBCReadOptions): + pass + + +class TeradataWriteOptions(JDBCWriteOptions): + pass + + +class TeradataSQLOptions(JDBCSQLOptions): + pass + + +class TeradataFetchOptions(JDBCFetchOptions): + pass + + +class TeradataExecuteOptions(JDBCExecuteOptions): + pass