[DOP-13855] Fix Clickhouse HWM tests

MobileTeleSystems · Apr 12, 2024 · 4535c3d · 4535c3d
1 parent 3b07f37
commit 4535c3d
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 0 deletions.
diff --git a/tests/fixtures/processing/clickhouse.py b/tests/fixtures/processing/clickhouse.py
@@ -152,3 +152,19 @@ def get_expected_dataframe(
         order_by: str | None = None,
     ) -> pandas.DataFrame:
         return self.connection.query_dataframe(self.get_expected_dataframe_ddl(schema, table, order_by))
+
+    def fix_pandas_df(
+        self,
+        df: pandas.DataFrame,
+    ) -> pandas.DataFrame:
+        df = super().fix_pandas_df(df)
+
+        for column in df.columns:
+            column_name = column.lower()
+
+            if "float" in column_name:
+                # somethere in chain Clickhouse -> Spark -> Pandas Float32 column is being converted to Float64,
+                # causing tests to fail. Convert it back to original type
+                df[column] = df[column].astype("float32")
+
+        return df
diff --git a/..._integration/tests_incremental_strategy_integration/test_strategy_increment_clickhouse.py b/..._integration/tests_incremental_strategy_integration/test_strategy_increment_clickhouse.py
@@ -308,6 +308,24 @@ def test_clickhouse_strategy_incremental_explicit_hwm_type(
             ColumnDateTimeHWM,
             lambda x: x.isoformat(),
         ),
+        (
+            "hwm_datetime",
+            "CAST(text_string AS DateTime32)",
+            ColumnDateTimeHWM,
+            lambda x: x.isoformat(),
+        ),
+        (
+            "hwm_datetime",
+            "CAST(text_string AS DateTime64)",
+            ColumnDateTimeHWM,
+            lambda x: x.isoformat(),
+        ),
+        (
+            "hwm_datetime",
+            "CAST(text_string AS DateTime64(6))",
+            ColumnDateTimeHWM,
+            lambda x: x.isoformat(),
+        ),
     ],
 )
 def test_clickhouse_strategy_incremental_with_hwm_expr(