diff --git a/df_to_azure/__init__.py b/df_to_azure/__init__.py index d376ed7..ec51cef 100644 --- a/df_to_azure/__init__.py +++ b/df_to_azure/__init__.py @@ -2,7 +2,7 @@ from .export import df_to_azure as df_to_azure -__version__ = "1.0.0" +__version__ = "1.0.1" logging.basicConfig( format="%(asctime)s.%(msecs)03d [%(levelname)-5s] [%(name)s] - %(message)s", diff --git a/df_to_azure/export.py b/df_to_azure/export.py index 137b273..8de94fe 100644 --- a/df_to_azure/export.py +++ b/df_to_azure/export.py @@ -188,7 +188,7 @@ def upload_to_blob(self): datetime_dtypes = self.df.select_dtypes("datetime") if datetime_dtypes.empty is False: for col in datetime_dtypes.columns: - self.df[col] = self.df[col].astype(str) + self.df[col] = self.df[col].astype(str).replace("NaT", None) data = self.df.to_parquet(index=False) blob_client.upload_blob(data, overwrite=True) diff --git a/df_to_azure/tests/test_general.py b/df_to_azure/tests/test_general.py index 17ee998..96c3c55 100644 --- a/df_to_azure/tests/test_general.py +++ b/df_to_azure/tests/test_general.py @@ -3,7 +3,7 @@ import pytest from keyvault import secrets_to_environment from numpy import array, nan -from pandas import DataFrame, Series, date_range, read_sql_query, read_sql_table +from pandas import DataFrame, Series, date_range, read_sql_query, read_sql_table, NaT from pandas._testing import assert_frame_equal from df_to_azure import df_to_azure @@ -44,6 +44,8 @@ def test_mapping_column_types(): "Categorical": Series(["a", "b", "c"], dtype="category"), } ) + df["Date_with_nat"] = df["Date"] + df["Date_with_nat"].iloc[2] = NaT df_to_azure( df, tablename="test_df_to_azure", @@ -67,6 +69,7 @@ def test_mapping_column_types(): "Float", "Float32", "Date", + "Date", "Timedelta", "Bool", "Categorical", @@ -83,6 +86,7 @@ def test_mapping_column_types(): "numeric", "numeric", "datetime", + "datetime", "numeric", "bit", "varchar", diff --git a/setup.cfg b/setup.cfg index aed4f49..d39c1ad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = df_to_azure -version = 1.0.0 +version = 1.0.1 author = Zypp author_email = hello@zypp.io description = Automatically write pandas DataFrames to SQL by creating pipelines in Azure Data Factory with copy activity from blob to SQL