From de225a2819952918c9b2d843fe268b3f9b4fe332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Fri, 26 Apr 2024 08:19:40 +0000 Subject: [PATCH] [DOP-13853] Update MongoDB package to 10.2.3 --- docs/changelog/next_release/255.feature.rst | 2 +- .../connection/db_connection/mongodb/types.rst | 4 ++-- .../db_connection/mongodb/connection.py | 16 ++++++++-------- .../test_mongodb_unit.py | 18 +++++++++--------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/changelog/next_release/255.feature.rst b/docs/changelog/next_release/255.feature.rst index 55d780196..aff64c57c 100644 --- a/docs/changelog/next_release/255.feature.rst +++ b/docs/changelog/next_release/255.feature.rst @@ -1 +1 @@ -:class:`MongoDB` connection now uses MongoDB Spark connector ``10.2.2``, upgraded from ``10.1.1``, and supports passing custom versions: ``MongoDB.get_packages(scala_version=..., package_version=...)``. +:class:`MongoDB` connection now uses MongoDB Spark connector ``10.2.3``, upgraded from ``10.1.1``, and supports passing custom versions: ``MongoDB.get_packages(scala_version=..., package_version=...)``. diff --git a/docs/connection/db_connection/mongodb/types.rst b/docs/connection/db_connection/mongodb/types.rst index bd5978aa9..f9787ff2e 100644 --- a/docs/connection/db_connection/mongodb/types.rst +++ b/docs/connection/db_connection/mongodb/types.rst @@ -73,8 +73,8 @@ References Here you can find source code with type conversions: -* `MongoDB -> Spark `_ -* `Spark -> MongoDB `_ +* `MongoDB -> Spark `_ +* `Spark -> MongoDB `_ Supported types --------------- diff --git a/onetl/connection/db_connection/mongodb/connection.py b/onetl/connection/db_connection/mongodb/connection.py index 5d4299f39..f2c959295 100644 --- a/onetl/connection/db_connection/mongodb/connection.py +++ b/onetl/connection/db_connection/mongodb/connection.py @@ -50,8 +50,8 @@ class Config: class MongoDB(DBConnection): """MongoDB connection. |support_hooks| - Based on package ``org.mongodb.spark:mongo-spark-connector:10.1.1`` - (`MongoDB connector for Spark `_) + Based on package `org.mongodb.spark:mongo-spark-connector:10.2.3 `_ + (`MongoDB connector for Spark `_) .. warning:: @@ -149,7 +149,7 @@ def get_packages( Spark version in format ``major.minor``. Used only if ``scala_version=None``. package_version : str, optional - Specifies the version of the MongoDB Spark connector to use. Defaults to ``10.2.2``. + Specifies the version of the MongoDB Spark connector to use. Defaults to ``10.2.3``. Examples -------- @@ -160,10 +160,10 @@ def get_packages( MongoDB.get_packages(scala_version="2.12") # specify custom connector version - MongoDB.get_packages(scala_version="2.12", package_version="10.2.2") + MongoDB.get_packages(scala_version="2.12", package_version="10.2.3") """ - default_package_version = "10.2.2" + default_package_version = "10.2.3" if scala_version: scala_ver = Version(scala_version).min_digits(2) @@ -190,7 +190,7 @@ def package_spark_3_2(cls) -> str: "use `MongoDB.get_packages(spark_version='3.2')` instead" ) warnings.warn(msg, UserWarning, stacklevel=3) - return "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2" + return "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3" @classproperty def package_spark_3_3(cls) -> str: @@ -200,7 +200,7 @@ def package_spark_3_3(cls) -> str: "use `MongoDB.get_packages(spark_version='3.3')` instead" ) warnings.warn(msg, UserWarning, stacklevel=3) - return "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2" + return "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3" @classproperty def package_spark_3_4(cls) -> str: @@ -210,7 +210,7 @@ def package_spark_3_4(cls) -> str: "use `MongoDB.get_packages(spark_version='3.4')` instead" ) warnings.warn(msg, UserWarning, stacklevel=3) - return "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2" + return "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3" @slot def pipeline( diff --git a/tests/tests_unit/tests_db_connection_unit/test_mongodb_unit.py b/tests/tests_unit/tests_db_connection_unit/test_mongodb_unit.py index 5333617b1..3e5f85215 100644 --- a/tests/tests_unit/tests_db_connection_unit/test_mongodb_unit.py +++ b/tests/tests_unit/tests_db_connection_unit/test_mongodb_unit.py @@ -12,9 +12,9 @@ def test_mongodb_package(): warning_msg = re.escape("will be removed in 1.0.0, use `MongoDB.get_packages(spark_version=") with pytest.warns(UserWarning, match=warning_msg): - assert MongoDB.package_spark_3_2 == "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2" - assert MongoDB.package_spark_3_3 == "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2" - assert MongoDB.package_spark_3_4 == "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2" + assert MongoDB.package_spark_3_2 == "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3" + assert MongoDB.package_spark_3_3 == "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3" + assert MongoDB.package_spark_3_4 == "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3" def test_mongodb_get_packages_no_input(): @@ -50,16 +50,16 @@ def test_mongodb_get_packages_scala_version_not_supported(scala_version): @pytest.mark.parametrize( "spark_version, scala_version, package_version, package", [ - (None, "2.12", "10.2.2", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2"), - (None, "2.13", "10.2.2", "org.mongodb.spark:mongo-spark-connector_2.13:10.2.2"), - ("3.2", None, "10.2.2", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2"), - ("3.3", None, "10.2.2", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2"), - ("3.4", None, "10.2.2", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2"), + (None, "2.12", "10.2.3", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3"), + (None, "2.13", "10.2.3", "org.mongodb.spark:mongo-spark-connector_2.13:10.2.3"), + ("3.2", None, "10.2.3", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3"), + ("3.3", None, "10.2.3", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3"), + ("3.4", None, "10.2.3", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3"), ("3.2", "2.12", "10.1.1", "org.mongodb.spark:mongo-spark-connector_2.12:10.1.1"), ("3.4", "2.13", "10.1.1", "org.mongodb.spark:mongo-spark-connector_2.13:10.1.1"), ("3.2", "2.12", "10.2.1", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.1"), ("3.2", "2.12", "10.2.0", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.0"), - ("3.2.4", "2.12.1", "10.2.2", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.2"), + ("3.2.4", "2.12.1", "10.2.3", "org.mongodb.spark:mongo-spark-connector_2.12:10.2.3"), ], ) def test_mongodb_get_packages(spark_version, scala_version, package_version, package):