Merge pull request #118 from zypp-io/development

Release 0.8.0

df_to_azure/__init__.py

@@ -2,7 +2,7 @@
 
 from .export import df_to_azure
 
-__version__ = "0.7.0-rc.2"
+__version__ = "0.8.0"
 
 logging.basicConfig(
     format="%(asctime)s.%(msecs)03d [%(levelname)-5s] [%(name)s] - %(message)s",

df_to_azure/export.py

@@ -119,6 +119,10 @@ def run(self):
         if self.clean_staging & (self.method == "upsert"):
             # If you used clean_staging=False before and the upsert gives errors on unknown columns -> remove table in
             # staging manually
+            if not self.wait_till_finished:
+                # Only remove after pipeline is done
+                logging.info("Wait until pipeline is done before cleaning staging")
+                wait_until_pipeline_is_done(self.adf_client, run_response)
             self.clean_staging_after_upsert()
 
         return self.adf_client, run_response

requirements.txt

@@ -1,8 +1,8 @@
-azure-identity==1.6.0
-azure-mgmt-datafactory==1.1.0
-azure-mgmt-resource==19.0.0
-azure-storage-blob==12.8.1
-pandas==1.3.2
-pyarrow==5.0.0
-pyodbc==4.0.31
-sqlalchemy==1.4.22
+azure-identity~=1.7.1
+azure-mgmt-datafactory~=2.2.0
+azure-mgmt-resource~=20.1.0
+azure-storage-blob~=12.8.1
+pandas~=1.4.1
+pyarrow~=7.0.0
+pyodbc~=4.0.32
+sqlalchemy~=1.4.31

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = df_to_azure
-version = 0.7.0-rc.2
+version = 0.8.0
 author = Melvin Folkers, Erfan Nariman
 author_email = melvin@zypp.io, erfan@zypp.io
 description = Automatically write pandas DataFrames to SQL by creating pipelines in Azure Data Factory with copy activity from blob to SQL
@@ -20,14 +20,14 @@ classifiers =
 packages = df_to_azure
 python_requires = >=3.7
 install_requires =
-    azure-identity==1.6.0
-    azure-mgmt-datafactory==1.1.0
-    azure-mgmt-resource==19.0.0
-    azure-storage-blob==12.8.1
-    pandas==1.3.2
-    pyarrow==5.0.0
-    pyodbc==4.0.31
-    sqlalchemy==1.4.22
+    azure-identity~=1.7.1
+    azure-mgmt-datafactory~=2.2.0
+    azure-mgmt-resource~=20.1.0
+    azure-storage-blob~=12.8.1
+    pandas~=1.4.1
+    pyarrow~=7.0.0
+    pyodbc~=4.0.32
+    sqlalchemy~=1.4.31
 
 
 [flake8]