[DOP-13855] - update clickhouse get_packages() tests

MobileTeleSystems · Apr 12, 2024 · 740639e · 740639e
1 parent 82b5042
commit 740639e
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 28 deletions.
diff --git a/onetl/connection/db_connection/clickhouse/connection.py b/onetl/connection/db_connection/clickhouse/connection.py
@@ -6,6 +6,7 @@
 from typing import ClassVar, Optional
 
 from onetl._util.classproperty import classproperty
+from onetl._util.version import Version
 from onetl.connection.db_connection.clickhouse.dialect import ClickhouseDialect
 from onetl.connection.db_connection.jdbc_connection import JDBCConnection
 from onetl.connection.db_connection.jdbc_mixin import JDBCStatementType
@@ -130,26 +131,29 @@ def get_packages(
 
             from onetl.connection import Clickhouse
 
-            Clickhouse.get_packages("0.7.1", "5.4")
+            Clickhouse.get_packages(package_version="0.7.1", apache_http_client_version="5.4")
 
         .. note::
 
-            Spark does not support ``.jar`` classifiers, so it is not possible to pass
-            ``com.clickhouse:clickhouse-jdbc:0.6.0-all`` to install all required packages.
-            Dependencies are listed manually.
+             Spark does not support ``.jar`` classifiers, so it is not possible to pass
+             ``com.clickhouse:clickhouse-jdbc:0.6.0:all`` to install all required packages.
 
         """
-        package_version = package_version or "0.6.0"
-        apache_http_client_version = apache_http_client_version or "5.3.1"
+        package_version_obj = Version(package_version) if package_version else Version("0.6.0")
+        apache_http_client_version_obj = (
+            Version(apache_http_client_version) if apache_http_client_version else Version("5.3.1")
+        )
+        if len(package_version_obj) != 3 or len(apache_http_client_version_obj) != 3:
+            raise ValueError("Version should consist of exactly three numeric_parts (major.minor.patch)")
 
         result = [
-            f"com.clickhouse:clickhouse-jdbc:{package_version}",
-            f"com.clickhouse:clickhouse-http-client:{package_version}",
+            f"com.clickhouse:clickhouse-jdbc:{package_version_obj}",
+            f"com.clickhouse:clickhouse-http-client:{package_version_obj}",
         ]
 
-        if package_version >= "0.5.0":
+        if package_version_obj >= Version("0.5.0"):
             #  before 0.5.0 builtin Java HTTP Client was used
-            result.append(f"org.apache.httpcomponents.client5:httpclient5:{apache_http_client_version}")
+            result.append(f"org.apache.httpcomponents.client5:httpclient5:{apache_http_client_version_obj}")
 
         return result
 

diff --git a/tests/tests_integration/tests_db_connection_integration/test_clickhouse_integration.py b/tests/tests_integration/tests_db_connection_integration/test_clickhouse_integration.py
@@ -227,6 +227,8 @@ def table_finalizer():
     updated_df = pandas.concat([updated_rows, unchanged_rows])
     processing.assert_equal_df(df=df, other_frame=updated_df, order_by="id_int")
 
+    clickhouse.execute(f"UPDATE {temp_table} SET hwm_int = 1 WHERE id_int < 50{suffix}")
+
     clickhouse.execute(f"ALTER TABLE {temp_table} DELETE WHERE id_int < 70{suffix}")
     df = clickhouse.fetch(f"SELECT * FROM {temp_table}{suffix}")
     assert df.count()

diff --git a/..._integration/tests_incremental_strategy_integration/test_strategy_increment_clickhouse.py b/..._integration/tests_incremental_strategy_integration/test_strategy_increment_clickhouse.py
@@ -304,7 +304,7 @@ def test_clickhouse_strategy_incremental_explicit_hwm_type(
         ),
         (
             "hwm_datetime",
-            "CAST(text_string AS DateTime64(6))",
+            "CAST(text_string AS DateTime)",
             ColumnDateTimeHWM,
             lambda x: x.isoformat(),
         ),

diff --git a/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py b/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py
@@ -14,22 +14,59 @@ def test_clickhouse_package():
     assert Clickhouse.package == expected_packages
 
 
-def test_clickhouse_get_packages():
-    expected_packages = [
-        "com.clickhouse:clickhouse-jdbc:0.6.0",
-        "com.clickhouse:clickhouse-http-client:0.6.0",
-        "org.apache.httpcomponents.client5:httpclient5:5.3.1",
-    ]
-    assert Clickhouse.get_packages() == expected_packages
-
-
-def test_clickhouse_get_packages_custom():
-    expected_packages = [
-        "com.clickhouse:clickhouse-jdbc:0.7.1",
-        "com.clickhouse:clickhouse-http-client:0.7.1",
-        "org.apache.httpcomponents.client5:httpclient5:5.4",
-    ]
-    assert Clickhouse.get_packages("0.7.1", "5.4") == expected_packages
+@pytest.mark.parametrize(
+    "package_version, apache_http_client_version, expected_packages",
+    [
+        (
+            None,
+            None,
+            [
+                "com.clickhouse:clickhouse-jdbc:0.6.0",
+                "com.clickhouse:clickhouse-http-client:0.6.0",
+                "org.apache.httpcomponents.client5:httpclient5:5.3.1",
+            ],
+        ),
+        (
+            "0.7.1",
+            "0.6.0",
+            [
+                "com.clickhouse:clickhouse-jdbc:0.7.1",
+                "com.clickhouse:clickhouse-http-client:0.7.1",
+                "org.apache.httpcomponents.client5:httpclient5:0.6.0",
+            ],
+        ),
+        (
+            "0.6.0-patch3",
+            "5.3.1",
+            [
+                "com.clickhouse:clickhouse-jdbc:0.6.0-patch3",
+                "com.clickhouse:clickhouse-http-client:0.6.0-patch3",
+                "org.apache.httpcomponents.client5:httpclient5:5.3.1",
+            ],
+        ),
+    ],
+)
+def test_clickhouse_get_packages(package_version, apache_http_client_version, expected_packages):
+    assert (
+        Clickhouse.get_packages(package_version=package_version, apache_http_client_version=apache_http_client_version)
+        == expected_packages
+    )
+
+
+@pytest.mark.parametrize(
+    "package_version, apache_http_client_version",
+    [
+        ("0.7", "5.3.1"),
+        ("1", "5.4.0"),
+        ("a.b.c", "5.3.1"),
+    ],
+)
+def test_invalid_versions_raise_error(package_version, apache_http_client_version):
+    with pytest.raises(
+        ValueError,
+        match=r"version should consist of exactly three numeric_parts \(major\.minor\.patch\)",
+    ):
+        Clickhouse.get_packages(package_version=package_version, apache_http_client_version=apache_http_client_version)
 
 
 def test_clickhouse_missing_package(spark_no_packages):

diff --git a/tests/util/assert_df.py b/tests/util/assert_df.py
@@ -67,4 +67,4 @@ def assert_subset_df(
 
     for column in columns:  # noqa: WPS528
         difference = ~small_pdf[column].isin(large_pdf[column])
-        assert not difference.all(), large_pdf[difference]
+        assert not difference.all(), small_pdf[difference]