diff --git a/docs/changelog/next_release/277.feature.rst b/docs/changelog/next_release/277.feature.rst deleted file mode 100644 index d9f871c57..000000000 --- a/docs/changelog/next_release/277.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``ReadOptions``, ``WriteOptions``, ``SQLOptions``, ``FetchOptions``, ``ExecuteOptions`` option classes for each JDBC connector. diff --git a/tests/tests_unit/tests_db_connection_unit/test_db_options_unit.py b/tests/tests_unit/tests_db_connection_unit/test_db_options_unit.py index bc280ee69..a280d53c7 100644 --- a/tests/tests_unit/tests_db_connection_unit/test_db_options_unit.py +++ b/tests/tests_unit/tests_db_connection_unit/test_db_options_unit.py @@ -3,7 +3,16 @@ import pytest -from onetl.connection import Greenplum, Hive, Postgres +from onetl.connection import ( + MSSQL, + Clickhouse, + Greenplum, + Hive, + MySQL, + Oracle, + Postgres, + Teradata, +) pytestmark = [pytest.mark.postgres] @@ -18,6 +27,26 @@ Postgres.Options, Greenplum.ReadOptions, Greenplum.WriteOptions, + Clickhouse.ReadOptions, + Clickhouse.WriteOptions, + Clickhouse.FetchOptions, + Clickhouse.ExecuteOptions, + MSSQL.ReadOptions, + MSSQL.WriteOptions, + MSSQL.FetchOptions, + MSSQL.ExecuteOptions, + MySQL.ReadOptions, + MySQL.WriteOptions, + MySQL.FetchOptions, + MySQL.ExecuteOptions, + Teradata.ReadOptions, + Teradata.WriteOptions, + Teradata.FetchOptions, + Teradata.ExecuteOptions, + Oracle.ReadOptions, + Oracle.WriteOptions, + Oracle.FetchOptions, + Oracle.ExecuteOptions, ], ) @pytest.mark.parametrize( @@ -44,6 +73,16 @@ def test_db_options_connection_parameters_cannot_be_passed(options_class, arg, v (Postgres.Options, "JDBCLegacyOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}), (Greenplum.ReadOptions, "GreenplumReadOptions", {"partitions": 10}), (Greenplum.WriteOptions, "GreenplumWriteOptions", {"if_exists": "replace_entire_table"}), + (Clickhouse.ReadOptions, "ClickhouseReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}), + (Clickhouse.WriteOptions, "ClickhouseWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}), + (MSSQL.ReadOptions, "MSSQLReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}), + (MSSQL.WriteOptions, "MSSQLWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}), + (MySQL.ReadOptions, "MySQLReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}), + (MySQL.WriteOptions, "MySQLWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}), + (Teradata.ReadOptions, "TeradataReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}), + (Teradata.WriteOptions, "TeradataWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}), + (Oracle.ReadOptions, "OracleReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}), + (Oracle.WriteOptions, "OracleWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}), ], ) def test_db_options_warn_for_unknown(options_class, options_class_name, known_options, caplog): @@ -65,18 +104,20 @@ def test_db_options_warn_for_unknown(options_class, options_class_name, known_op @pytest.mark.parametrize( "options_class,options", [ - ( - Postgres.ReadOptions, - Postgres.WriteOptions(), - ), - ( - Postgres.WriteOptions, - Postgres.ReadOptions(), - ), - ], - ids=[ - "Write options object passed to ReadOptions", - "Read options object passed to WriteOptions", + (Postgres.ReadOptions, Postgres.WriteOptions()), + (Postgres.WriteOptions, Postgres.ReadOptions()), + (Clickhouse.ReadOptions, Clickhouse.WriteOptions()), + (Clickhouse.WriteOptions, Clickhouse.ReadOptions()), + (MSSQL.ReadOptions, MSSQL.WriteOptions()), + (MSSQL.WriteOptions, MSSQL.ReadOptions()), + (MySQL.ReadOptions, MySQL.WriteOptions()), + (MySQL.WriteOptions, MySQL.ReadOptions()), + (Teradata.ReadOptions, Teradata.WriteOptions()), + (Teradata.WriteOptions, Teradata.ReadOptions()), + (Greenplum.ReadOptions, Greenplum.WriteOptions()), + (Greenplum.WriteOptions, Greenplum.ReadOptions()), + (Oracle.ReadOptions, Oracle.WriteOptions()), + (Oracle.WriteOptions, Oracle.ReadOptions()), ], ) def test_db_options_parse_mismatch_class(options_class, options): @@ -106,15 +147,36 @@ def test_db_options_parse_mismatch_connection_and_options_types(connection, opti @pytest.mark.parametrize( "options_class", [ + # PostgreSQL options Postgres.ReadOptions, Postgres.WriteOptions, Postgres.FetchOptions, Postgres.ExecuteOptions, + Postgres.Options, Greenplum.ReadOptions, Greenplum.WriteOptions, Hive.WriteOptions, - Postgres.Options, Hive.Options, + Clickhouse.ReadOptions, + Clickhouse.WriteOptions, + Clickhouse.FetchOptions, + Clickhouse.ExecuteOptions, + MSSQL.ReadOptions, + MSSQL.WriteOptions, + MSSQL.FetchOptions, + MSSQL.ExecuteOptions, + MySQL.ReadOptions, + MySQL.WriteOptions, + MySQL.FetchOptions, + MySQL.ExecuteOptions, + Teradata.ReadOptions, + Teradata.WriteOptions, + Teradata.FetchOptions, + Teradata.ExecuteOptions, + Oracle.ReadOptions, + Oracle.WriteOptions, + Oracle.FetchOptions, + Oracle.ExecuteOptions, ], ) @pytest.mark.parametrize( diff --git a/tests/tests_unit/tests_db_connection_unit/test_jdbc_options_unit.py b/tests/tests_unit/tests_db_connection_unit/test_jdbc_options_unit.py index 690c17092..699838889 100644 --- a/tests/tests_unit/tests_db_connection_unit/test_jdbc_options_unit.py +++ b/tests/tests_unit/tests_db_connection_unit/test_jdbc_options_unit.py @@ -3,7 +3,7 @@ import pytest from onetl._internal import to_camel -from onetl.connection import Postgres +from onetl.connection import MSSQL, Clickhouse, MySQL, Oracle, Postgres, Teradata from onetl.connection.db_connection.jdbc_connection import JDBCTableExistBehavior pytestmark = [pytest.mark.postgres] @@ -44,21 +44,57 @@ def test_jdbc_options_default(): ("properties", {"abc": "cde"}), ], ) -@pytest.mark.parametrize("options_class", [Postgres.FetchOptions, Postgres.ExecuteOptions]) -def test_jdbc_read_write_options_populated_by_connection_class(arg, value, options_class): - error_msg = rf"Options \['{arg}'\] are not allowed to use in a PostgresReadOptions" - with pytest.raises(ValueError, match=error_msg): - Postgres.ReadOptions.parse({arg: value}) - - error_msg = rf"Options \['{arg}'\] are not allowed to use in a PostgresWriteOptions" - with pytest.raises(ValueError, match=error_msg): - Postgres.WriteOptions.parse({arg: value}) - - # FetchOptions & ExecuteOptions does not have such restriction - options = options_class.parse({arg: value}) - assert options.dict()[arg] == value +@pytest.mark.parametrize( + "options_class, read_write_restriction", + [ + (Postgres.FetchOptions, False), + (Postgres.ExecuteOptions, False), + (Postgres.ReadOptions, True), + (Postgres.WriteOptions, True), + (Clickhouse.FetchOptions, False), + (Clickhouse.ExecuteOptions, False), + (Clickhouse.ReadOptions, True), + (Clickhouse.WriteOptions, True), + (MSSQL.FetchOptions, False), + (MSSQL.ExecuteOptions, False), + (MSSQL.ReadOptions, True), + (MSSQL.WriteOptions, True), + (MySQL.FetchOptions, False), + (MySQL.ExecuteOptions, False), + (MySQL.ReadOptions, True), + (MySQL.WriteOptions, True), + (Teradata.FetchOptions, False), + (Teradata.ExecuteOptions, False), + (Teradata.ReadOptions, True), + (Teradata.WriteOptions, True), + (Oracle.FetchOptions, False), + (Oracle.ExecuteOptions, False), + (Oracle.ReadOptions, True), + (Oracle.WriteOptions, True), + ], +) +def test_jdbc_read_write_options_populated_by_connection_class(arg, value, options_class, read_write_restriction): + if read_write_restriction: + error_msg = rf"Options \['{arg}'\] are not allowed to use in a {options_class.__name__}" + with pytest.raises(ValueError, match=error_msg): + options_class.parse({arg: value}) + else: + # FetchOptions & ExecuteOptions does not have such restriction + options = options_class.parse({arg: value}) + assert options.dict()[arg] == value +@pytest.mark.parametrize( + "options_class, options_class_name", + [ + (Postgres.ReadOptions, "PostgresReadOptions"), + (Clickhouse.ReadOptions, "ClickhouseReadOptions"), + (MSSQL.ReadOptions, "MSSQLReadOptions"), + (MySQL.ReadOptions, "MySQLReadOptions"), + (Teradata.ReadOptions, "TeradataReadOptions"), + (Oracle.ReadOptions, "OracleReadOptions"), + ], +) @pytest.mark.parametrize( "arg, value", [ @@ -73,12 +109,23 @@ def test_jdbc_read_write_options_populated_by_connection_class(arg, value, optio ("createTableColumnTypes", "a varchar"), ], ) -def test_jdbc_write_options_cannot_be_used_in_read_options(arg, value): - error_msg = rf"Options \['{arg}'\] are not allowed to use in a PostgresReadOptions" +def test_jdbc_write_options_cannot_be_used_in_read_options(arg, value, options_class, options_class_name): + error_msg = rf"Options \['{arg}'\] are not allowed to use in a {options_class_name}" with pytest.raises(ValueError, match=error_msg): - Postgres.ReadOptions.parse({arg: value}) + options_class.parse({arg: value}) +@pytest.mark.parametrize( + "options_class, options_class_name", + [ + (Postgres.WriteOptions, "PostgresWriteOptions"), + (Clickhouse.WriteOptions, "ClickhouseWriteOptions"), + (MSSQL.WriteOptions, "MSSQLWriteOptions"), + (MySQL.WriteOptions, "MySQLWriteOptions"), + (Teradata.WriteOptions, "TeradataWriteOptions"), + (Oracle.WriteOptions, "OracleWriteOptions"), + ], +) @pytest.mark.parametrize( "arg, value", [ @@ -101,10 +148,10 @@ def test_jdbc_write_options_cannot_be_used_in_read_options(arg, value): ("predicates", "s"), ], ) -def test_jdbc_read_options_cannot_be_used_in_write_options(arg, value): - error_msg = rf"Options \['{arg}'\] are not allowed to use in a PostgresWriteOptions" +def test_jdbc_read_options_cannot_be_used_in_write_options(options_class, options_class_name, arg, value): + error_msg = rf"Options \['{arg}'\] are not allowed to use in a {options_class_name}" with pytest.raises(ValueError, match=error_msg): - Postgres.WriteOptions.parse({arg: value}) + options_class.parse({arg: value}) @pytest.mark.parametrize( @@ -137,12 +184,23 @@ def test_jdbc_old_options_allowed_but_deprecated(arg, value): assert options.dict(by_alias=True)[to_camel(arg)] == value -def test_jdbc_read_options_partitioning_is_not_valid(): +@pytest.mark.parametrize( + "options_class", + [ + Postgres.ReadOptions, + Clickhouse.ReadOptions, + MSSQL.ReadOptions, + MySQL.ReadOptions, + Teradata.ReadOptions, + Oracle.ReadOptions, + ], +) +def test_jdbc_read_options_partitioning_is_not_valid(options_class): with pytest.raises(ValueError): - Postgres.ReadOptions(numPartitions=200) + options_class(numPartitions=200) with pytest.raises(ValueError): - Postgres.ReadOptions(partitionColumn="test") + options_class(partitionColumn="test") def test_jdbc_read_options_case(): @@ -254,14 +312,19 @@ def test_jdbc_write_options_mode_deprecated(options, value, message): @pytest.mark.parametrize( - "options", + "options_class, options", [ - {"mode": "wrong_mode"}, + (Postgres.WriteOptions, {"mode": "wrong_mode"}), + (Clickhouse.WriteOptions, {"mode": "wrong_mode"}), + (MSSQL.WriteOptions, {"mode": "wrong_mode"}), + (MySQL.WriteOptions, {"mode": "wrong_mode"}), + (Teradata.WriteOptions, {"mode": "wrong_mode"}), + (Oracle.WriteOptions, {"mode": "wrong_mode"}), ], ) -def test_jdbc_write_options_mode_wrong(options): +def test_jdbc_write_options_mode_wrong(options_class, options): with pytest.raises(ValueError, match="value is not a valid enumeration member"): - Postgres.WriteOptions(**options) + options_class(**options) @pytest.mark.parametrize( @@ -277,13 +340,35 @@ def test_jdbc_sql_options_partition_bounds(options, expected_message): Postgres.SQLOptions(**options) -def test_jdbc_sql_options_partitioning_mode_prohibited(): +@pytest.mark.parametrize( + "options_class", + [ + Postgres.SQLOptions, + Clickhouse.SQLOptions, + MSSQL.SQLOptions, + MySQL.SQLOptions, + Teradata.SQLOptions, + Oracle.SQLOptions, + ], +) +def test_jdbc_sql_options_partitioning_mode_prohibited(options_class): with pytest.raises(ValueError, match=r"Options \['partitioning_mode'\] are not allowed"): - Postgres.SQLOptions(partitioning_mode="range") + options_class(partitioning_mode="range") -def test_jdbc_sql_options_default(): - options = Postgres.SQLOptions() +@pytest.mark.parametrize( + "options_class", + [ + Postgres.SQLOptions, + Clickhouse.SQLOptions, + MSSQL.SQLOptions, + MySQL.SQLOptions, + Teradata.SQLOptions, + Oracle.SQLOptions, + ], +) +def test_jdbc_sql_options_default(options_class): + options = options_class() assert options.fetchsize == 100_000 assert options.query_timeout is None