From b8c52316f2cecd0daabe287af4a537695345bff3 Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Thu, 16 Mar 2023 15:21:23 +0100 Subject: [PATCH 1/6] Up package version --- requirements.txt | 6 +++--- setup.cfg | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index f9cfdbd..95006dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ azure-identity~=1.7.1 azure-mgmt-datafactory~=2.2.0 azure-mgmt-resource~=20.1.0 -azure-storage-blob~=12.8.1 -pandas~=1.4.1 -pyarrow~=7.0.0 +azure-storage-blob~=12.15.0 +pandas~=1.5.3 +pyarrow~=11.0.0 pyodbc~=4.0.32 sqlalchemy~=1.4.31 diff --git a/setup.cfg b/setup.cfg index b8edf86..5827838 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,9 +23,9 @@ install_requires = azure-identity~=1.7.1 azure-mgmt-datafactory~=2.2.0 azure-mgmt-resource~=20.1.0 - azure-storage-blob~=12.8.1 - pandas~=1.4.1 - pyarrow~=7.0.0 + azure-storage-blob~=12.15.0 + pandas~=1.5.3 + pyarrow~=11.0.0 pyodbc~=4.0.32 sqlalchemy~=1.4.31 From 40898c5d43988507ffeb9fcca5e1b47a4136143b Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 17 Mar 2023 08:23:44 +0100 Subject: [PATCH 2/6] Loosen requirements --- requirements.txt | 16 ++++++++-------- setup.cfg | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/requirements.txt b/requirements.txt index 95006dc..a2b7aa0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -azure-identity~=1.7.1 -azure-mgmt-datafactory~=2.2.0 -azure-mgmt-resource~=20.1.0 -azure-storage-blob~=12.15.0 -pandas~=1.5.3 -pyarrow~=11.0.0 -pyodbc~=4.0.32 -sqlalchemy~=1.4.31 +azure-identity>=1.7.1 +azure-mgmt-datafactory>=2.2.0 +azure-mgmt-resource>=20.1.0 +azure-storage-blob>=12.8.1 +pandas>=1.4.1 +pyarrow>=7.0.0 +pyodbc>=4.0.32 +sqlalchemy>=1.4.31 diff --git a/setup.cfg b/setup.cfg index 5827838..f4bdd8b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,14 +20,14 @@ classifiers = packages = df_to_azure python_requires = >=3.7 install_requires = - azure-identity~=1.7.1 - azure-mgmt-datafactory~=2.2.0 - azure-mgmt-resource~=20.1.0 - azure-storage-blob~=12.15.0 - pandas~=1.5.3 - pyarrow~=11.0.0 - pyodbc~=4.0.32 - sqlalchemy~=1.4.31 + azure-identity>=1.7.1 + azure-mgmt-datafactory>=2.2.0 + azure-mgmt-resource>=20.1.0 + azure-storage-blob>=12.8.1 + pandas>=1.4.1 + pyarrow>=7.0.0 + pyodbc>=4.0.32 + sqlalchemy>=1.4.31 [flake8] From 9003ed7dd7b483fa54a003a9bd499516d33046b4 Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 17 Mar 2023 15:07:10 +0100 Subject: [PATCH 3/6] Fix requirements --- requirements.txt | 4 ++-- setup.cfg | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index a2b7aa0..50c0a9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ azure-identity>=1.7.1 -azure-mgmt-datafactory>=2.2.0 +azure-mgmt-datafactory>=2.2.0,<2.7.0 azure-mgmt-resource>=20.1.0 azure-storage-blob>=12.8.1 pandas>=1.4.1 pyarrow>=7.0.0 pyodbc>=4.0.32 -sqlalchemy>=1.4.31 +sqlalchemy>=1.4.31,<2.0.0 diff --git a/setup.cfg b/setup.cfg index f4bdd8b..2cf4d73 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,13 +21,14 @@ packages = df_to_azure python_requires = >=3.7 install_requires = azure-identity>=1.7.1 - azure-mgmt-datafactory>=2.2.0 + azure-mgmt-datafactory>=2.2.0,<2.7.0 azure-mgmt-resource>=20.1.0 azure-storage-blob>=12.8.1 pandas>=1.4.1 pyarrow>=7.0.0 pyodbc>=4.0.32 - sqlalchemy>=1.4.31 + sqlalchemy>=1.4.31,<2.0.0 + [flake8] From 878b7a3c3c27ed0283dcd756d3d5ffb51ff41bcb Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 17 Mar 2023 20:02:55 +0100 Subject: [PATCH 4/6] Fix test and replace deprecated parameter --- df_to_azure/db.py | 4 ++-- df_to_azure/export.py | 2 +- df_to_azure/tests/test_general.py | 32 +++++++++---------------------- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/df_to_azure/db.py b/df_to_azure/db.py index 8abaddd..2c1e6b8 100644 --- a/df_to_azure/db.py +++ b/df_to_azure/db.py @@ -71,14 +71,14 @@ def create_stored_procedure(self): ) -def auth_azure(): +def auth_azure(driver: str = "ODBC Driver 17 for SQL Server"): connection_string = "mssql+pyodbc://{}:{}@{}:1433/{}?driver={}".format( os.environ.get("SQL_USER"), quote_plus(os.environ.get("SQL_PW")), os.environ.get("SQL_SERVER"), os.environ.get("SQL_DB"), - "ODBC Driver 17 for SQL Server", + driver, ) con = create_engine(connection_string).connect() diff --git a/df_to_azure/export.py b/df_to_azure/export.py index b54920c..e1af7a5 100644 --- a/df_to_azure/export.py +++ b/df_to_azure/export.py @@ -197,7 +197,7 @@ def upload_to_blob(self): blob=f"{self.table_name}/{self.table_name}.csv", ) - data = self.df.to_csv(index=False, sep="^", quotechar='"', line_terminator="\n") + data = self.df.to_csv(index=False, sep="^", quotechar='"', lineterminator="\n") blob_client.upload_blob(data, overwrite=True) def create_schema(self): diff --git a/df_to_azure/tests/test_general.py b/df_to_azure/tests/test_general.py index e67af4d..ac7df81 100644 --- a/df_to_azure/tests/test_general.py +++ b/df_to_azure/tests/test_general.py @@ -161,14 +161,15 @@ def test_pipeline_name(): def test_empty_dataframe(): df = DataFrame() - with pytest.raises(SystemExit): - df_to_azure( - df=df, - tablename="empty_dataframe", - schema="test", - method="create", - wait_till_finished=True, - ) + adf_client, run_response = df_to_azure( + df=df, + tablename="empty_dataframe", + schema="test", + method="create", + wait_till_finished=True, + ) + assert adf_client is None + assert run_response is None def test_convert_bigint(): @@ -203,18 +204,3 @@ def test_double_column_names(): schema="test", wait_till_finished=True, ) - - -def test_dataframe_with_no_data(): - - # scenario where there are 2 dataframes, first one empty and the second one with data. - df_list = [DataFrame(), DataFrame({"A": [1, 2, 3], "B": [10, 20, 30], "C": ["X", "Y", "Z"]})] - - results = list() - - for df in df_list: - r = df_to_azure(df, tablename=f"dataset_{df.shape[0]}_records", schema="test") - results.append(r) - - # there should be 2 results from the 2 df_to_azure operations. - assert len(results) == 2 From 179b4e5f77a4b1da896b33e04f2ee631ca933f4c Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 17 Mar 2023 20:05:03 +0100 Subject: [PATCH 5/6] Up version number --- df_to_azure/__init__.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/df_to_azure/__init__.py b/df_to_azure/__init__.py index 8eda3c1..cb22d33 100644 --- a/df_to_azure/__init__.py +++ b/df_to_azure/__init__.py @@ -2,7 +2,7 @@ from .export import df_to_azure -__version__ = "0.8.1" +__version__ = "0.9.0" logging.basicConfig( format="%(asctime)s.%(msecs)03d [%(levelname)-5s] [%(name)s] - %(message)s", diff --git a/setup.cfg b/setup.cfg index 2cf4d73..e863c66 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = df_to_azure -version = 0.8.1 +version = 0.9.0 author = Melvin Folkers, Erfan Nariman author_email = melvin@zypp.io, erfan@zypp.io description = Automatically write pandas DataFrames to SQL by creating pipelines in Azure Data Factory with copy activity from blob to SQL From 5fb1cdfb4636cc332675a5a7ab393df276a35419 Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 24 Mar 2023 08:34:08 +0100 Subject: [PATCH 6/6] Set pandas --- requirements.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 50c0a9b..6c846fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ azure-identity>=1.7.1 azure-mgmt-datafactory>=2.2.0,<2.7.0 azure-mgmt-resource>=20.1.0 azure-storage-blob>=12.8.1 -pandas>=1.4.1 +pandas>=1.5.0 pyarrow>=7.0.0 pyodbc>=4.0.32 sqlalchemy>=1.4.31,<2.0.0 diff --git a/setup.cfg b/setup.cfg index e863c66..5c71130 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ install_requires = azure-mgmt-datafactory>=2.2.0,<2.7.0 azure-mgmt-resource>=20.1.0 azure-storage-blob>=12.8.1 - pandas>=1.4.1 + pandas>=1.5.0 pyarrow>=7.0.0 pyodbc>=4.0.32 sqlalchemy>=1.4.31,<2.0.0