Merge pull request #132 from zypp-io/fix-bug-NaT

Fix bug NaT

df_to_azure/__init__.py

@@ -2,7 +2,7 @@
 
 from .export import df_to_azure as df_to_azure
 
-__version__ = "1.0.0"
+__version__ = "1.0.1"
 
 logging.basicConfig(
     format="%(asctime)s.%(msecs)03d [%(levelname)-5s] [%(name)s] - %(message)s",

df_to_azure/export.py

@@ -188,7 +188,7 @@ def upload_to_blob(self):
         datetime_dtypes = self.df.select_dtypes("datetime")
         if datetime_dtypes.empty is False:
             for col in datetime_dtypes.columns:
-                self.df[col] = self.df[col].astype(str)
+                self.df[col] = self.df[col].astype(str).replace("NaT", None)
         data = self.df.to_parquet(index=False)
         blob_client.upload_blob(data, overwrite=True)
 

df_to_azure/tests/test_general.py

@@ -3,7 +3,7 @@
 import pytest
 from keyvault import secrets_to_environment
 from numpy import array, nan
-from pandas import DataFrame, Series, date_range, read_sql_query, read_sql_table
+from pandas import DataFrame, Series, date_range, read_sql_query, read_sql_table, NaT
 from pandas._testing import assert_frame_equal
 
 from df_to_azure import df_to_azure
@@ -44,6 +44,8 @@ def test_mapping_column_types():
             "Categorical": Series(["a", "b", "c"], dtype="category"),
         }
     )
+    df["Date_with_nat"] = df["Date"]
+    df["Date_with_nat"].iloc[2] = NaT
     df_to_azure(
         df,
         tablename="test_df_to_azure",
@@ -67,6 +69,7 @@ def test_mapping_column_types():
                 "Float",
                 "Float32",
                 "Date",
+                "Date",
                 "Timedelta",
                 "Bool",
                 "Categorical",
@@ -83,6 +86,7 @@ def test_mapping_column_types():
                 "numeric",
                 "numeric",
                 "datetime",
+                "datetime",
                 "numeric",
                 "bit",
                 "varchar",

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = df_to_azure
-version = 1.0.0
+version = 1.0.1
 author = Zypp
 author_email = hello@zypp.io
 description = Automatically write pandas DataFrames to SQL by creating pipelines in Azure Data Factory with copy activity from blob to SQL