diff --git a/onetl/connection/db_connection/clickhouse/connection.py b/onetl/connection/db_connection/clickhouse/connection.py index efa6c7edc..f46a2e176 100644 --- a/onetl/connection/db_connection/clickhouse/connection.py +++ b/onetl/connection/db_connection/clickhouse/connection.py @@ -6,6 +6,7 @@ from typing import ClassVar, Optional from onetl._util.classproperty import classproperty +from onetl._util.version import Version from onetl.connection.db_connection.clickhouse.dialect import ClickhouseDialect from onetl.connection.db_connection.jdbc_connection import JDBCConnection from onetl.connection.db_connection.jdbc_mixin import JDBCStatementType @@ -130,26 +131,29 @@ def get_packages( from onetl.connection import Clickhouse - Clickhouse.get_packages("0.7.1", "5.4") + Clickhouse.get_packages(package_version="0.7.1", apache_http_client_version="5.4") .. note:: - Spark does not support ``.jar`` classifiers, so it is not possible to pass - ``com.clickhouse:clickhouse-jdbc:0.6.0-all`` to install all required packages. - Dependencies are listed manually. + Spark does not support ``.jar`` classifiers, so it is not possible to pass + ``com.clickhouse:clickhouse-jdbc:0.6.0:all`` to install all required packages. """ - package_version = package_version or "0.6.0" - apache_http_client_version = apache_http_client_version or "5.3.1" + package_version_obj = Version(package_version) if package_version else Version("0.6.0") + apache_http_client_version_obj = ( + Version(apache_http_client_version) if apache_http_client_version else Version("5.3.1") + ) + if len(package_version_obj) != 3 or len(apache_http_client_version_obj) != 3: + raise ValueError("Version should consist of exactly three numeric_parts (major.minor.patch)") result = [ - f"com.clickhouse:clickhouse-jdbc:{package_version}", - f"com.clickhouse:clickhouse-http-client:{package_version}", + f"com.clickhouse:clickhouse-jdbc:{package_version_obj}", + f"com.clickhouse:clickhouse-http-client:{package_version_obj}", ] - if package_version >= "0.5.0": + if package_version_obj >= Version("0.5.0"): # before 0.5.0 builtin Java HTTP Client was used - result.append(f"org.apache.httpcomponents.client5:httpclient5:{apache_http_client_version}") + result.append(f"org.apache.httpcomponents.client5:httpclient5:{apache_http_client_version_obj}") return result diff --git a/tests/tests_integration/tests_db_connection_integration/test_clickhouse_integration.py b/tests/tests_integration/tests_db_connection_integration/test_clickhouse_integration.py index 27052a9f1..c786b1fe0 100644 --- a/tests/tests_integration/tests_db_connection_integration/test_clickhouse_integration.py +++ b/tests/tests_integration/tests_db_connection_integration/test_clickhouse_integration.py @@ -227,6 +227,8 @@ def table_finalizer(): updated_df = pandas.concat([updated_rows, unchanged_rows]) processing.assert_equal_df(df=df, other_frame=updated_df, order_by="id_int") + clickhouse.execute(f"UPDATE {temp_table} SET hwm_int = 1 WHERE id_int < 50{suffix}") + clickhouse.execute(f"ALTER TABLE {temp_table} DELETE WHERE id_int < 70{suffix}") df = clickhouse.fetch(f"SELECT * FROM {temp_table}{suffix}") assert df.count() diff --git a/tests/tests_integration/tests_strategy_integration/tests_incremental_strategy_integration/test_strategy_increment_clickhouse.py b/tests/tests_integration/tests_strategy_integration/tests_incremental_strategy_integration/test_strategy_increment_clickhouse.py index 478a8e380..77ec071b3 100644 --- a/tests/tests_integration/tests_strategy_integration/tests_incremental_strategy_integration/test_strategy_increment_clickhouse.py +++ b/tests/tests_integration/tests_strategy_integration/tests_incremental_strategy_integration/test_strategy_increment_clickhouse.py @@ -304,7 +304,7 @@ def test_clickhouse_strategy_incremental_explicit_hwm_type( ), ( "hwm_datetime", - "CAST(text_string AS DateTime64(6))", + "CAST(text_string AS DateTime)", ColumnDateTimeHWM, lambda x: x.isoformat(), ), diff --git a/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py b/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py index 708388417..346cf9cd4 100644 --- a/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py +++ b/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py @@ -14,22 +14,59 @@ def test_clickhouse_package(): assert Clickhouse.package == expected_packages -def test_clickhouse_get_packages(): - expected_packages = [ - "com.clickhouse:clickhouse-jdbc:0.6.0", - "com.clickhouse:clickhouse-http-client:0.6.0", - "org.apache.httpcomponents.client5:httpclient5:5.3.1", - ] - assert Clickhouse.get_packages() == expected_packages - - -def test_clickhouse_get_packages_custom(): - expected_packages = [ - "com.clickhouse:clickhouse-jdbc:0.7.1", - "com.clickhouse:clickhouse-http-client:0.7.1", - "org.apache.httpcomponents.client5:httpclient5:5.4", - ] - assert Clickhouse.get_packages("0.7.1", "5.4") == expected_packages +@pytest.mark.parametrize( + "package_version, apache_http_client_version, expected_packages", + [ + ( + None, + None, + [ + "com.clickhouse:clickhouse-jdbc:0.6.0", + "com.clickhouse:clickhouse-http-client:0.6.0", + "org.apache.httpcomponents.client5:httpclient5:5.3.1", + ], + ), + ( + "0.7.1", + "0.6.0", + [ + "com.clickhouse:clickhouse-jdbc:0.7.1", + "com.clickhouse:clickhouse-http-client:0.7.1", + "org.apache.httpcomponents.client5:httpclient5:0.6.0", + ], + ), + ( + "0.6.0-patch3", + "5.3.1", + [ + "com.clickhouse:clickhouse-jdbc:0.6.0-patch3", + "com.clickhouse:clickhouse-http-client:0.6.0-patch3", + "org.apache.httpcomponents.client5:httpclient5:5.3.1", + ], + ), + ], +) +def test_clickhouse_get_packages(package_version, apache_http_client_version, expected_packages): + assert ( + Clickhouse.get_packages(package_version=package_version, apache_http_client_version=apache_http_client_version) + == expected_packages + ) + + +@pytest.mark.parametrize( + "package_version, apache_http_client_version", + [ + ("0.7", "5.3.1"), + ("1", "5.4.0"), + ("a.b.c", "5.3.1"), + ], +) +def test_invalid_versions_raise_error(package_version, apache_http_client_version): + with pytest.raises( + ValueError, + match=r"version should consist of exactly three numeric_parts \(major\.minor\.patch\)", + ): + Clickhouse.get_packages(package_version=package_version, apache_http_client_version=apache_http_client_version) def test_clickhouse_missing_package(spark_no_packages): diff --git a/tests/util/assert_df.py b/tests/util/assert_df.py index f7adad032..e36c9af73 100644 --- a/tests/util/assert_df.py +++ b/tests/util/assert_df.py @@ -67,4 +67,4 @@ def assert_subset_df( for column in columns: # noqa: WPS528 difference = ~small_pdf[column].isin(large_pdf[column]) - assert not difference.all(), large_pdf[difference] + assert not difference.all(), small_pdf[difference]