Skip to content

Commit

Permalink
[DOP-15764] - add parametrized tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@maxim-lixakov
maxim-lixakov committed May 22, 2024
1 parent c18a04b commit 393b16f
Show file tree
Hide file tree
Showing 3 changed files with 192 additions and 46 deletions.
1 change: 0 additions & 1 deletion docs/changelog/next_release/277.feature.rst
View file

This file was deleted.

90 changes: 76 additions & 14 deletions tests/tests_unit/tests_db_connection_unit/test_db_options_unit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,16 @@

import pytest

from onetl.connection import Greenplum, Hive, Postgres
from onetl.connection import (
MSSQL,
Clickhouse,
Greenplum,
Hive,
MySQL,
Oracle,
Postgres,
Teradata,
)

pytestmark = [pytest.mark.postgres]

Expand All @@ -18,6 +27,26 @@
Postgres.Options,
Greenplum.ReadOptions,
Greenplum.WriteOptions,
Clickhouse.ReadOptions,
Clickhouse.WriteOptions,
Clickhouse.FetchOptions,
Clickhouse.ExecuteOptions,
MSSQL.ReadOptions,
MSSQL.WriteOptions,
MSSQL.FetchOptions,
MSSQL.ExecuteOptions,
MySQL.ReadOptions,
MySQL.WriteOptions,
MySQL.FetchOptions,
MySQL.ExecuteOptions,
Teradata.ReadOptions,
Teradata.WriteOptions,
Teradata.FetchOptions,
Teradata.ExecuteOptions,
Oracle.ReadOptions,
Oracle.WriteOptions,
Oracle.FetchOptions,
Oracle.ExecuteOptions,
],
)
@pytest.mark.parametrize(
Expand All @@ -44,6 +73,16 @@ def test_db_options_connection_parameters_cannot_be_passed(options_class, arg, v
(Postgres.Options, "JDBCLegacyOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}),
(Greenplum.ReadOptions, "GreenplumReadOptions", {"partitions": 10}),
(Greenplum.WriteOptions, "GreenplumWriteOptions", {"if_exists": "replace_entire_table"}),
(Clickhouse.ReadOptions, "ClickhouseReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}),
(Clickhouse.WriteOptions, "ClickhouseWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}),
(MSSQL.ReadOptions, "MSSQLReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}),
(MSSQL.WriteOptions, "MSSQLWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}),
(MySQL.ReadOptions, "MySQLReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}),
(MySQL.WriteOptions, "MySQLWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}),
(Teradata.ReadOptions, "TeradataReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}),
(Teradata.WriteOptions, "TeradataWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}),
(Oracle.ReadOptions, "OracleReadOptions", {"fetchsize": 10, "keytab": "a/b/c"}),
(Oracle.WriteOptions, "OracleWriteOptions", {"if_exists": "replace_entire_table", "keytab": "a/b/c"}),
],
)
def test_db_options_warn_for_unknown(options_class, options_class_name, known_options, caplog):
Expand All @@ -65,18 +104,20 @@ def test_db_options_warn_for_unknown(options_class, options_class_name, known_op
@pytest.mark.parametrize(
"options_class,options",
[
(
Postgres.ReadOptions,
Postgres.WriteOptions(),
),
(
Postgres.WriteOptions,
Postgres.ReadOptions(),
),
],
ids=[
"Write options object passed to ReadOptions",
"Read options object passed to WriteOptions",
(Postgres.ReadOptions, Postgres.WriteOptions()),
(Postgres.WriteOptions, Postgres.ReadOptions()),
(Clickhouse.ReadOptions, Clickhouse.WriteOptions()),
(Clickhouse.WriteOptions, Clickhouse.ReadOptions()),
(MSSQL.ReadOptions, MSSQL.WriteOptions()),
(MSSQL.WriteOptions, MSSQL.ReadOptions()),
(MySQL.ReadOptions, MySQL.WriteOptions()),
(MySQL.WriteOptions, MySQL.ReadOptions()),
(Teradata.ReadOptions, Teradata.WriteOptions()),
(Teradata.WriteOptions, Teradata.ReadOptions()),
(Greenplum.ReadOptions, Greenplum.WriteOptions()),
(Greenplum.WriteOptions, Greenplum.ReadOptions()),
(Oracle.ReadOptions, Oracle.WriteOptions()),
(Oracle.WriteOptions, Oracle.ReadOptions()),
],
)
def test_db_options_parse_mismatch_class(options_class, options):
Expand Down Expand Up @@ -106,15 +147,36 @@ def test_db_options_parse_mismatch_connection_and_options_types(connection, opti
@pytest.mark.parametrize(
"options_class",
[
# PostgreSQL options
Postgres.ReadOptions,
Postgres.WriteOptions,
Postgres.FetchOptions,
Postgres.ExecuteOptions,
Postgres.Options,
Greenplum.ReadOptions,
Greenplum.WriteOptions,
Hive.WriteOptions,
Postgres.Options,
Hive.Options,
Clickhouse.ReadOptions,
Clickhouse.WriteOptions,
Clickhouse.FetchOptions,
Clickhouse.ExecuteOptions,
MSSQL.ReadOptions,
MSSQL.WriteOptions,
MSSQL.FetchOptions,
MSSQL.ExecuteOptions,
MySQL.ReadOptions,
MySQL.WriteOptions,
MySQL.FetchOptions,
MySQL.ExecuteOptions,
Teradata.ReadOptions,
Teradata.WriteOptions,
Teradata.FetchOptions,
Teradata.ExecuteOptions,
Oracle.ReadOptions,
Oracle.WriteOptions,
Oracle.FetchOptions,
Oracle.ExecuteOptions,
],
)
@pytest.mark.parametrize(
Expand Down
147 changes: 116 additions & 31 deletions tests/tests_unit/tests_db_connection_unit/test_jdbc_options_unit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import pytest

from onetl._internal import to_camel
from onetl.connection import Postgres
from onetl.connection import MSSQL, Clickhouse, MySQL, Oracle, Postgres, Teradata
from onetl.connection.db_connection.jdbc_connection import JDBCTableExistBehavior

pytestmark = [pytest.mark.postgres]
Expand Down Expand Up @@ -44,21 +44,57 @@ def test_jdbc_options_default():
("properties", {"abc": "cde"}),
],
)
@pytest.mark.parametrize("options_class", [Postgres.FetchOptions, Postgres.ExecuteOptions])
def test_jdbc_read_write_options_populated_by_connection_class(arg, value, options_class):
error_msg = rf"Options \['{arg}'\] are not allowed to use in a PostgresReadOptions"
with pytest.raises(ValueError, match=error_msg):
Postgres.ReadOptions.parse({arg: value})

error_msg = rf"Options \['{arg}'\] are not allowed to use in a PostgresWriteOptions"
with pytest.raises(ValueError, match=error_msg):
Postgres.WriteOptions.parse({arg: value})

# FetchOptions & ExecuteOptions does not have such restriction
options = options_class.parse({arg: value})
assert options.dict()[arg] == value
@pytest.mark.parametrize(
"options_class, read_write_restriction",
[
(Postgres.FetchOptions, False),
(Postgres.ExecuteOptions, False),
(Postgres.ReadOptions, True),
(Postgres.WriteOptions, True),
(Clickhouse.FetchOptions, False),
(Clickhouse.ExecuteOptions, False),
(Clickhouse.ReadOptions, True),
(Clickhouse.WriteOptions, True),
(MSSQL.FetchOptions, False),
(MSSQL.ExecuteOptions, False),
(MSSQL.ReadOptions, True),
(MSSQL.WriteOptions, True),
(MySQL.FetchOptions, False),
(MySQL.ExecuteOptions, False),
(MySQL.ReadOptions, True),
(MySQL.WriteOptions, True),
(Teradata.FetchOptions, False),
(Teradata.ExecuteOptions, False),
(Teradata.ReadOptions, True),
(Teradata.WriteOptions, True),
(Oracle.FetchOptions, False),
(Oracle.ExecuteOptions, False),
(Oracle.ReadOptions, True),
(Oracle.WriteOptions, True),
],
)
def test_jdbc_read_write_options_populated_by_connection_class(arg, value, options_class, read_write_restriction):
if read_write_restriction:
error_msg = rf"Options \['{arg}'\] are not allowed to use in a {options_class.__name__}"
with pytest.raises(ValueError, match=error_msg):
options_class.parse({arg: value})
else:
# FetchOptions & ExecuteOptions does not have such restriction
options = options_class.parse({arg: value})
assert options.dict()[arg] == value


@pytest.mark.parametrize(
"options_class, options_class_name",
[
(Postgres.ReadOptions, "PostgresReadOptions"),
(Clickhouse.ReadOptions, "ClickhouseReadOptions"),
(MSSQL.ReadOptions, "MSSQLReadOptions"),
(MySQL.ReadOptions, "MySQLReadOptions"),
(Teradata.ReadOptions, "TeradataReadOptions"),
(Oracle.ReadOptions, "OracleReadOptions"),
],
)
@pytest.mark.parametrize(
"arg, value",
[
Expand All @@ -73,12 +109,23 @@ def test_jdbc_read_write_options_populated_by_connection_class(arg, value, optio
("createTableColumnTypes", "a varchar"),
],
)
def test_jdbc_write_options_cannot_be_used_in_read_options(arg, value):
error_msg = rf"Options \['{arg}'\] are not allowed to use in a PostgresReadOptions"
def test_jdbc_write_options_cannot_be_used_in_read_options(arg, value, options_class, options_class_name):
error_msg = rf"Options \['{arg}'\] are not allowed to use in a {options_class_name}"
with pytest.raises(ValueError, match=error_msg):
Postgres.ReadOptions.parse({arg: value})
options_class.parse({arg: value})


@pytest.mark.parametrize(
"options_class, options_class_name",
[
(Postgres.WriteOptions, "PostgresWriteOptions"),
(Clickhouse.WriteOptions, "ClickhouseWriteOptions"),
(MSSQL.WriteOptions, "MSSQLWriteOptions"),
(MySQL.WriteOptions, "MySQLWriteOptions"),
(Teradata.WriteOptions, "TeradataWriteOptions"),
(Oracle.WriteOptions, "OracleWriteOptions"),
],
)
@pytest.mark.parametrize(
"arg, value",
[
Expand All @@ -101,10 +148,10 @@ def test_jdbc_write_options_cannot_be_used_in_read_options(arg, value):
("predicates", "s"),
],
)
def test_jdbc_read_options_cannot_be_used_in_write_options(arg, value):
error_msg = rf"Options \['{arg}'\] are not allowed to use in a PostgresWriteOptions"
def test_jdbc_read_options_cannot_be_used_in_write_options(options_class, options_class_name, arg, value):
error_msg = rf"Options \['{arg}'\] are not allowed to use in a {options_class_name}"
with pytest.raises(ValueError, match=error_msg):
Postgres.WriteOptions.parse({arg: value})
options_class.parse({arg: value})


@pytest.mark.parametrize(
Expand Down Expand Up @@ -137,12 +184,23 @@ def test_jdbc_old_options_allowed_but_deprecated(arg, value):
assert options.dict(by_alias=True)[to_camel(arg)] == value


def test_jdbc_read_options_partitioning_is_not_valid():
@pytest.mark.parametrize(
"options_class",
[
Postgres.ReadOptions,
Clickhouse.ReadOptions,
MSSQL.ReadOptions,
MySQL.ReadOptions,
Teradata.ReadOptions,
Oracle.ReadOptions,
],
)
def test_jdbc_read_options_partitioning_is_not_valid(options_class):
with pytest.raises(ValueError):
Postgres.ReadOptions(numPartitions=200)
options_class(numPartitions=200)

with pytest.raises(ValueError):
Postgres.ReadOptions(partitionColumn="test")
options_class(partitionColumn="test")


def test_jdbc_read_options_case():
Expand Down Expand Up @@ -254,14 +312,19 @@ def test_jdbc_write_options_mode_deprecated(options, value, message):


@pytest.mark.parametrize(
"options",
"options_class, options",
[
{"mode": "wrong_mode"},
(Postgres.WriteOptions, {"mode": "wrong_mode"}),
(Clickhouse.WriteOptions, {"mode": "wrong_mode"}),
(MSSQL.WriteOptions, {"mode": "wrong_mode"}),
(MySQL.WriteOptions, {"mode": "wrong_mode"}),
(Teradata.WriteOptions, {"mode": "wrong_mode"}),
(Oracle.WriteOptions, {"mode": "wrong_mode"}),
],
)
def test_jdbc_write_options_mode_wrong(options):
def test_jdbc_write_options_mode_wrong(options_class, options):
with pytest.raises(ValueError, match="value is not a valid enumeration member"):
Postgres.WriteOptions(**options)
options_class(**options)


@pytest.mark.parametrize(
Expand All @@ -277,13 +340,35 @@ def test_jdbc_sql_options_partition_bounds(options, expected_message):
Postgres.SQLOptions(**options)


def test_jdbc_sql_options_partitioning_mode_prohibited():
@pytest.mark.parametrize(
"options_class",
[
Postgres.SQLOptions,
Clickhouse.SQLOptions,
MSSQL.SQLOptions,
MySQL.SQLOptions,
Teradata.SQLOptions,
Oracle.SQLOptions,
],
)
def test_jdbc_sql_options_partitioning_mode_prohibited(options_class):
with pytest.raises(ValueError, match=r"Options \['partitioning_mode'\] are not allowed"):
Postgres.SQLOptions(partitioning_mode="range")
options_class(partitioning_mode="range")


def test_jdbc_sql_options_default():
options = Postgres.SQLOptions()
@pytest.mark.parametrize(
"options_class",
[
Postgres.SQLOptions,
Clickhouse.SQLOptions,
MSSQL.SQLOptions,
MySQL.SQLOptions,
Teradata.SQLOptions,
Oracle.SQLOptions,
],
)
def test_jdbc_sql_options_default(options_class):
options = options_class()
assert options.fetchsize == 100_000
assert options.query_timeout is None

Expand Down

0 comments on commit 393b16f

Please sign in to comment.