From 461a139cf2d555c19736248e172e2eb14496aec3 Mon Sep 17 00:00:00 2001 From: maxim-lixakov Date: Tue, 16 Apr 2024 08:35:56 +0300 Subject: [PATCH 1/2] [DOP-13849] - upgrade postgres packages --- docs/changelog/next_release/249.feature.rst | 2 +- docs/changelog/next_release/251.feature.rst | 1 + .../db_connection/clickhouse/connection.py | 30 +++++++------ .../db_connection/postgres/connection.py | 23 +++++++--- .../test_clickhouse_unit.py | 21 +++++++++- .../test_postgres_unit.py | 42 +++++++++++++++---- 6 files changed, 91 insertions(+), 28 deletions(-) create mode 100644 docs/changelog/next_release/251.feature.rst diff --git a/docs/changelog/next_release/249.feature.rst b/docs/changelog/next_release/249.feature.rst index 7d79a6c87..8ec0686c2 100644 --- a/docs/changelog/next_release/249.feature.rst +++ b/docs/changelog/next_release/249.feature.rst @@ -1 +1 @@ -Allow passing custom JDBC driver version to ``Clickhouse.get_packages(package_version=...)``. +:class:`Clickhouse` connection supports passing custom versions: ``Clickhouse.get_packages(package_version=...)``. diff --git a/docs/changelog/next_release/251.feature.rst b/docs/changelog/next_release/251.feature.rst new file mode 100644 index 000000000..bc8d528a2 --- /dev/null +++ b/docs/changelog/next_release/251.feature.rst @@ -0,0 +1 @@ +:class:`Postgres` connection now uses PostgreSQL JDBC driver ``42.7.3``, upgraded from ``42.6.0``, and supports passing custom versions: ``Postgres.get_packages(package_version=...)``. diff --git a/onetl/connection/db_connection/clickhouse/connection.py b/onetl/connection/db_connection/clickhouse/connection.py index fa06d4daa..658df3238 100644 --- a/onetl/connection/db_connection/clickhouse/connection.py +++ b/onetl/connection/db_connection/clickhouse/connection.py @@ -110,18 +110,18 @@ class Clickhouse(JDBCConnection): @classmethod def get_packages( cls, - package_version: str | None = None, - apache_http_client_version: str | None = None, + package_version: str | Version | None = None, + apache_http_client_version: str | Version | None = None, ) -> list[str]: """ - Get package names to be downloaded by Spark. |support_hooks| + Get package names to be downloaded by Spark. Allows specifying custom JDBC and Apache HTTP Client versions. |support_hooks| Parameters ---------- - package_version : str , optional + package_version : str | Version, optional ClickHouse JDBC version client packages. Defaults to ``0.6.0``. - apache_http_client_version : str, optional + apache_http_client_version : str | Version, optional Apache HTTP Client version package. Defaults to ``5.3.1``. Examples @@ -139,19 +139,23 @@ def get_packages( ``com.clickhouse:clickhouse-jdbc:0.6.0:all`` to install all required packages. """ - package_version_obj = Version(package_version).min_digits(3) if package_version else Version("0.6.0") - apache_http_client_version_obj = ( - Version(apache_http_client_version).min_digits(3) if apache_http_client_version else Version("5.3.1") + default_jdbc_version = "0.6.0" + default_http_version = "5.3.1" + + jdbc_version = default_jdbc_version if package_version is None else str(Version(package_version).min_digits(3)) + http_version = ( + default_http_version + if apache_http_client_version is None + else str(Version(apache_http_client_version).min_digits(3)) ) result = [ - f"com.clickhouse:clickhouse-jdbc:{package_version_obj}", - f"com.clickhouse:clickhouse-http-client:{package_version_obj}", + f"com.clickhouse:clickhouse-jdbc:{jdbc_version}", + f"com.clickhouse:clickhouse-http-client:{jdbc_version}", ] - if package_version_obj >= Version("0.5.0"): - # before 0.5.0 builtin Java HTTP Client was used - result.append(f"org.apache.httpcomponents.client5:httpclient5:{apache_http_client_version_obj}") + if Version(jdbc_version) >= Version("0.5.0"): + result.append(f"org.apache.httpcomponents.client5:httpclient5:{http_version}") return result diff --git a/onetl/connection/db_connection/postgres/connection.py b/onetl/connection/db_connection/postgres/connection.py index a4d27bcea..59d67916b 100644 --- a/onetl/connection/db_connection/postgres/connection.py +++ b/onetl/connection/db_connection/postgres/connection.py @@ -6,6 +6,7 @@ from typing import ClassVar from onetl._util.classproperty import classproperty +from onetl._util.version import Version from onetl.connection.db_connection.jdbc_connection import JDBCConnection from onetl.connection.db_connection.jdbc_mixin.options import JDBCOptions from onetl.connection.db_connection.postgres.dialect import PostgresDialect @@ -27,7 +28,7 @@ class Config: class Postgres(JDBCConnection): """PostgreSQL JDBC connection. |support_hooks| - Based on Maven package ``org.postgresql:postgresql:42.6.0`` + Based on Maven package ``org.postgresql:postgresql:42.7.3`` (`official Postgres JDBC driver `_). .. warning:: @@ -105,9 +106,14 @@ class Postgres(JDBCConnection): @slot @classmethod - def get_packages(cls) -> list[str]: + def get_packages(cls, package_version: str | Version | None = None) -> list[str]: """ - Get package names to be downloaded by Spark. |support_hooks| + Get package names to be downloaded by Spark. Allows specifying a custom JDBC driver version. |support_hooks| + + Parameters + ---------- + package_version : str | Version, optional + Specifies the version of the PostgreSQL JDBC driver to use. Defaults to ``42.7.3``. Examples -------- @@ -118,15 +124,22 @@ def get_packages(cls) -> list[str]: Postgres.get_packages() + # custom package version + Postgres.get_packages(package_version="42.6.0") + """ - return ["org.postgresql:postgresql:42.6.0"] + + default_version = "42.7.3" + version = default_version if package_version is None else str(Version(package_version).min_digits(3)) + + return [f"org.postgresql:postgresql:{version}"] @classproperty def package(cls) -> str: """Get package name to be downloaded by Spark.""" msg = "`Postgres.package` will be removed in 1.0.0, use `Postgres.get_packages()` instead" warnings.warn(msg, UserWarning, stacklevel=3) - return "org.postgresql:postgresql:42.6.0" + return "org.postgresql:postgresql:42.7.3" @property def jdbc_url(self) -> str: diff --git a/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py b/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py index 15388ee7e..62d30e6b4 100644 --- a/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py +++ b/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py @@ -1,5 +1,6 @@ import pytest +from onetl._util.version import Version from onetl.connection import Clickhouse pytestmark = [pytest.mark.clickhouse, pytest.mark.db_connection, pytest.mark.connection] @@ -58,6 +59,24 @@ def test_clickhouse_package(): "org.apache.httpcomponents.client5:httpclient5:4.5.14", ], ), + ( + Version("0.5.0"), + Version("4.5.14"), + [ + "com.clickhouse:clickhouse-jdbc:0.5.0", + "com.clickhouse:clickhouse-http-client:0.5.0", + "org.apache.httpcomponents.client5:httpclient5:4.5.14", + ], + ), + ( + Version("0.6.0"), + Version("4.5.14"), + [ + "com.clickhouse:clickhouse-jdbc:0.6.0", + "com.clickhouse:clickhouse-http-client:0.6.0", + "org.apache.httpcomponents.client5:httpclient5:4.5.14", + ], + ), ], ) def test_clickhouse_get_packages(package_version, apache_http_client_version, expected_packages): @@ -75,7 +94,7 @@ def test_clickhouse_get_packages(package_version, apache_http_client_version, ex ("a.b.c", "5.3.1"), ], ) -def test_invalid_versions_raise_error(package_version, apache_http_client_version): +def test_clickhouse_invalid_version(package_version, apache_http_client_version): with pytest.raises( ValueError, match=rf"Version '{package_version}' does not have enough numeric components for requested format \(expected at least 3\).", diff --git a/tests/tests_unit/tests_db_connection_unit/test_postgres_unit.py b/tests/tests_unit/tests_db_connection_unit/test_postgres_unit.py index f6b5f4e9b..6f63d4d11 100644 --- a/tests/tests_unit/tests_db_connection_unit/test_postgres_unit.py +++ b/tests/tests_unit/tests_db_connection_unit/test_postgres_unit.py @@ -2,6 +2,7 @@ import pytest +from onetl._util.version import Version from onetl.connection import Postgres pytestmark = [pytest.mark.postgres, pytest.mark.db_connection, pytest.mark.connection] @@ -14,14 +15,39 @@ def test_postgres_class_attributes(): def test_postgres_package(): warning_msg = re.escape("will be removed in 1.0.0, use `Postgres.get_packages()` instead") with pytest.warns(UserWarning, match=warning_msg): - assert Postgres.package == "org.postgresql:postgresql:42.6.0" - - -def test_postgres_get_packages(): - assert Postgres.get_packages() == ["org.postgresql:postgresql:42.6.0"] - - -def test_oracle_missing_package(spark_no_packages): + assert Postgres.package == "org.postgresql:postgresql:42.7.3" + + +@pytest.mark.parametrize( + "package_version, expected_packages", + [ + (None, ["org.postgresql:postgresql:42.7.3"]), + ("42.7.3", ["org.postgresql:postgresql:42.7.3"]), + ("42.7.3-patch", ["org.postgresql:postgresql:42.7.3-patch"]), + ("42.6.0", ["org.postgresql:postgresql:42.6.0"]), + (Version("42.5.1"), ["org.postgresql:postgresql:42.5.1"]), + ], +) +def test_postgres_get_packages(package_version, expected_packages): + assert Postgres.get_packages(package_version=package_version) == expected_packages + + +@pytest.mark.parametrize( + "package_version", + [ + "42.2", + "abc", + ], +) +def test_postgres_invalid_version(package_version): + with pytest.raises( + ValueError, + match=rf"Version '{package_version}' does not have enough numeric components for requested format \(expected at least 3\).", + ): + Postgres.get_packages(package_version=package_version) + + +def test_postgres_missing_package(spark_no_packages): msg = "Cannot import Java class 'org.postgresql.Driver'" with pytest.raises(ValueError, match=msg): Postgres( From c0f314c1342c469bc93ed389487d682e48351dba Mon Sep 17 00:00:00 2001 From: maxim-lixakov Date: Tue, 16 Apr 2024 14:11:23 +0300 Subject: [PATCH 2/2] [DOP-13849] - put away Version annotations --- .../db_connection/clickhouse/connection.py | 18 +++++++--------- .../db_connection/postgres/connection.py | 7 +++---- .../test_clickhouse_unit.py | 21 +------------------ .../test_postgres_unit.py | 4 +--- 4 files changed, 12 insertions(+), 38 deletions(-) diff --git a/onetl/connection/db_connection/clickhouse/connection.py b/onetl/connection/db_connection/clickhouse/connection.py index 658df3238..288612f3c 100644 --- a/onetl/connection/db_connection/clickhouse/connection.py +++ b/onetl/connection/db_connection/clickhouse/connection.py @@ -110,18 +110,18 @@ class Clickhouse(JDBCConnection): @classmethod def get_packages( cls, - package_version: str | Version | None = None, - apache_http_client_version: str | Version | None = None, + package_version: str | None = None, + apache_http_client_version: str | None = None, ) -> list[str]: """ Get package names to be downloaded by Spark. Allows specifying custom JDBC and Apache HTTP Client versions. |support_hooks| Parameters ---------- - package_version : str | Version, optional + package_version : str, optional ClickHouse JDBC version client packages. Defaults to ``0.6.0``. - apache_http_client_version : str | Version, optional + apache_http_client_version : str, optional Apache HTTP Client version package. Defaults to ``5.3.1``. Examples @@ -142,19 +142,15 @@ def get_packages( default_jdbc_version = "0.6.0" default_http_version = "5.3.1" - jdbc_version = default_jdbc_version if package_version is None else str(Version(package_version).min_digits(3)) - http_version = ( - default_http_version - if apache_http_client_version is None - else str(Version(apache_http_client_version).min_digits(3)) - ) + jdbc_version = Version(package_version or default_jdbc_version).min_digits(3) + http_version = Version(apache_http_client_version or default_http_version).min_digits(3) result = [ f"com.clickhouse:clickhouse-jdbc:{jdbc_version}", f"com.clickhouse:clickhouse-http-client:{jdbc_version}", ] - if Version(jdbc_version) >= Version("0.5.0"): + if jdbc_version >= Version("0.5.0"): result.append(f"org.apache.httpcomponents.client5:httpclient5:{http_version}") return result diff --git a/onetl/connection/db_connection/postgres/connection.py b/onetl/connection/db_connection/postgres/connection.py index 59d67916b..80cddbc11 100644 --- a/onetl/connection/db_connection/postgres/connection.py +++ b/onetl/connection/db_connection/postgres/connection.py @@ -106,13 +106,13 @@ class Postgres(JDBCConnection): @slot @classmethod - def get_packages(cls, package_version: str | Version | None = None) -> list[str]: + def get_packages(cls, package_version: str | None = None) -> list[str]: """ Get package names to be downloaded by Spark. Allows specifying a custom JDBC driver version. |support_hooks| Parameters ---------- - package_version : str | Version, optional + package_version : str, optional Specifies the version of the PostgreSQL JDBC driver to use. Defaults to ``42.7.3``. Examples @@ -128,9 +128,8 @@ def get_packages(cls, package_version: str | Version | None = None) -> list[str] Postgres.get_packages(package_version="42.6.0") """ - default_version = "42.7.3" - version = default_version if package_version is None else str(Version(package_version).min_digits(3)) + version = Version(package_version or default_version).min_digits(3) return [f"org.postgresql:postgresql:{version}"] diff --git a/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py b/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py index 62d30e6b4..6b400b93e 100644 --- a/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py +++ b/tests/tests_unit/tests_db_connection_unit/test_clickhouse_unit.py @@ -1,6 +1,5 @@ import pytest -from onetl._util.version import Version from onetl.connection import Clickhouse pytestmark = [pytest.mark.clickhouse, pytest.mark.db_connection, pytest.mark.connection] @@ -59,24 +58,6 @@ def test_clickhouse_package(): "org.apache.httpcomponents.client5:httpclient5:4.5.14", ], ), - ( - Version("0.5.0"), - Version("4.5.14"), - [ - "com.clickhouse:clickhouse-jdbc:0.5.0", - "com.clickhouse:clickhouse-http-client:0.5.0", - "org.apache.httpcomponents.client5:httpclient5:4.5.14", - ], - ), - ( - Version("0.6.0"), - Version("4.5.14"), - [ - "com.clickhouse:clickhouse-jdbc:0.6.0", - "com.clickhouse:clickhouse-http-client:0.6.0", - "org.apache.httpcomponents.client5:httpclient5:4.5.14", - ], - ), ], ) def test_clickhouse_get_packages(package_version, apache_http_client_version, expected_packages): @@ -94,7 +75,7 @@ def test_clickhouse_get_packages(package_version, apache_http_client_version, ex ("a.b.c", "5.3.1"), ], ) -def test_clickhouse_invalid_version(package_version, apache_http_client_version): +def test_clickhouse_get_packages_invalid_version(package_version, apache_http_client_version): with pytest.raises( ValueError, match=rf"Version '{package_version}' does not have enough numeric components for requested format \(expected at least 3\).", diff --git a/tests/tests_unit/tests_db_connection_unit/test_postgres_unit.py b/tests/tests_unit/tests_db_connection_unit/test_postgres_unit.py index 6f63d4d11..f4c00f30f 100644 --- a/tests/tests_unit/tests_db_connection_unit/test_postgres_unit.py +++ b/tests/tests_unit/tests_db_connection_unit/test_postgres_unit.py @@ -2,7 +2,6 @@ import pytest -from onetl._util.version import Version from onetl.connection import Postgres pytestmark = [pytest.mark.postgres, pytest.mark.db_connection, pytest.mark.connection] @@ -25,7 +24,6 @@ def test_postgres_package(): ("42.7.3", ["org.postgresql:postgresql:42.7.3"]), ("42.7.3-patch", ["org.postgresql:postgresql:42.7.3-patch"]), ("42.6.0", ["org.postgresql:postgresql:42.6.0"]), - (Version("42.5.1"), ["org.postgresql:postgresql:42.5.1"]), ], ) def test_postgres_get_packages(package_version, expected_packages): @@ -39,7 +37,7 @@ def test_postgres_get_packages(package_version, expected_packages): "abc", ], ) -def test_postgres_invalid_version(package_version): +def test_postgres_get_packages_invalid_version(package_version): with pytest.raises( ValueError, match=rf"Version '{package_version}' does not have enough numeric components for requested format \(expected at least 3\).",