From b1d6d252156ac5b8bf3168e6e879df3c23785660 Mon Sep 17 00:00:00 2001 From: Melvin Folkers Date: Thu, 13 Oct 2022 17:52:29 +0200 Subject: [PATCH 01/17] wrote test with no data that fails --- df_to_azure/tests/test_general.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/df_to_azure/tests/test_general.py b/df_to_azure/tests/test_general.py index 6fe075f..580a00a 100644 --- a/df_to_azure/tests/test_general.py +++ b/df_to_azure/tests/test_general.py @@ -203,3 +203,22 @@ def test_double_column_names(): schema="test", wait_till_finished=True, ) + + +def test_dataframe_with_no_data(): + + # scenario where there are 2 dataframes, first one empty and the second one with data. + df_list = [DataFrame(), DataFrame({"A": [1, 2, 3], "B": [10, 20, 30], "C": ["X", "Y", "Z"]})] + + results = list() + + for df in df_list: + r = df_to_azure(df, tablename="no_data", schema="test") + results.append(r) + + # there should be 2 results from the 2 df_to_azure operations. + assert len(results) == 2 + + +if __name__ == "__main__": + test_dataframe_with_no_data() From 4d88c4d089a0cd0260438ca7a0e8642461516bdf Mon Sep 17 00:00:00 2001 From: Melvin Folkers Date: Thu, 13 Oct 2022 18:01:50 +0200 Subject: [PATCH 02/17] wrote test with no data that fails --- df_to_azure/tests/test_general.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/df_to_azure/tests/test_general.py b/df_to_azure/tests/test_general.py index 580a00a..e67af4d 100644 --- a/df_to_azure/tests/test_general.py +++ b/df_to_azure/tests/test_general.py @@ -213,12 +213,8 @@ def test_dataframe_with_no_data(): results = list() for df in df_list: - r = df_to_azure(df, tablename="no_data", schema="test") + r = df_to_azure(df, tablename=f"dataset_{df.shape[0]}_records", schema="test") results.append(r) # there should be 2 results from the 2 df_to_azure operations. assert len(results) == 2 - - -if __name__ == "__main__": - test_dataframe_with_no_data() From 052279bf1678b0ac92eca1e562738e70d2649f8a Mon Sep 17 00:00:00 2001 From: Melvin Folkers Date: Thu, 13 Oct 2022 18:02:18 +0200 Subject: [PATCH 03/17] return None if df empty --- df_to_azure/export.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/df_to_azure/export.py b/df_to_azure/export.py index d74f9b6..d46e07b 100644 --- a/df_to_azure/export.py +++ b/df_to_azure/export.py @@ -97,6 +97,11 @@ def __init__( self.clean_staging = clean_staging def run(self): + + if self.df.empty: + logging.info("Data empty, no new records to upload.") + return None, None + if self.create: # azure components @@ -134,10 +139,6 @@ def _checks(self): def upload_dataset(self): - if self.df.empty: - logging.info("Data empty, no new records to upload.") - sys.exit(1) - if self.method == "create": self.create_schema() self.push_to_azure() From 6cac83100834c899df7e0fd66c2f1b3f952d190f Mon Sep 17 00:00:00 2001 From: Melvin Folkers Date: Thu, 13 Oct 2022 18:05:47 +0200 Subject: [PATCH 04/17] fix for flake --- df_to_azure/export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/df_to_azure/export.py b/df_to_azure/export.py index d46e07b..29e85f1 100644 --- a/df_to_azure/export.py +++ b/df_to_azure/export.py @@ -1,6 +1,5 @@ import logging import os -import sys from datetime import datetime from io import BytesIO from typing import Union From 247ce087def2c5a329e75eb5661544c8870d6d28 Mon Sep 17 00:00:00 2001 From: Melvin Folkers Date: Thu, 16 Mar 2023 08:52:28 +0100 Subject: [PATCH 05/17] add Float64Dtype --- df_to_azure/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/df_to_azure/export.py b/df_to_azure/export.py index d74f9b6..24c4466 100644 --- a/df_to_azure/export.py +++ b/df_to_azure/export.py @@ -9,7 +9,7 @@ import pandas as pd from azure.storage.blob import BlobServiceClient from numpy import dtype -from pandas import BooleanDtype, DataFrame, Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype, StringDtype +from pandas import BooleanDtype, DataFrame, Float64Dtype, Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype, StringDtype from sqlalchemy.sql.visitors import VisitableType from sqlalchemy.types import BigInteger, Boolean, DateTime, Integer, Numeric, String @@ -234,6 +234,7 @@ def column_types(self) -> dict: Int16Dtype(): Integer(), Int32Dtype(): Integer(), Int64Dtype(): Integer(), + Float64Dtype(): numeric, dtype("float64"): numeric, dtype("float32"): numeric, dtype("float16"): numeric, From 487d9301c1348589ed5f678a67f6bd5a3a5f7e91 Mon Sep 17 00:00:00 2001 From: Melvin Folkers Date: Thu, 16 Mar 2023 09:08:47 +0100 Subject: [PATCH 06/17] updated precommit --- .pre-commit-config.yaml | 21 ++++++++++++++++++--- df_to_azure/export.py | 13 ++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1862b7d..06a1530 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,37 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.3.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files + - id: check-merge-conflict + - id: debug-statements + - id: detect-private-key + - id: name-tests-test + args: [--pytest-test-first] + - id: requirements-txt-fixer - repo: https://github.com/pycqa/flake8 - rev: 3.9.2 + rev: 5.0.4 hooks: - id: flake8 + args: ["--statistics", "--count", "--max-complexity=10", "--max-line-length=120", "--per-file-ignore=__init__.py: F401"] - repo: https://github.com/psf/black rev: 22.3.0 hooks: - id: black args: [--line-length=120] - repo: https://github.com/PyCQA/isort - rev: 5.9.1 + rev: 5.12.0 hooks: - id: isort args: ["--profile", "black", --line-length=120] +- repo: local + hooks: + - id: check-requirements + name: Check requirements + description: Check if requirements in setup.cfg and requirements.txt are equal + language: python + entry: python scripts/check_setupcfg_and_requirements_equal.py + pass_filenames: false diff --git a/df_to_azure/export.py b/df_to_azure/export.py index 24c4466..1a9c27d 100644 --- a/df_to_azure/export.py +++ b/df_to_azure/export.py @@ -9,7 +9,17 @@ import pandas as pd from azure.storage.blob import BlobServiceClient from numpy import dtype -from pandas import BooleanDtype, DataFrame, Float64Dtype, Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype, StringDtype +from pandas import ( + BooleanDtype, + DataFrame, + DatetimeTZDtype, + Float64Dtype, + Int8Dtype, + Int16Dtype, + Int32Dtype, + Int64Dtype, + StringDtype, +) from sqlalchemy.sql.visitors import VisitableType from sqlalchemy.types import BigInteger, Boolean, DateTime, Integer, Numeric, String @@ -241,6 +251,7 @@ def column_types(self) -> dict: dtype(" Date: Thu, 16 Mar 2023 09:09:28 +0100 Subject: [PATCH 07/17] fix for flake --- .gitignore | 2 +- data/employee_1.csv | 2 +- data/employee_2.csv | 2 +- data/employee_duplicate_keys_1.csv | 2 +- data/employee_duplicate_keys_2.csv | 2 +- data/sample_1.csv | 2 +- data/sample_2.csv | 2 +- df_to_azure/settings.py | 1 + df_to_azure/utils.py | 2 +- 9 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 8fc82a5..9ed7c48 100644 --- a/.gitignore +++ b/.gitignore @@ -143,4 +143,4 @@ notebooks/ settings.yml # mac -.DS_Store \ No newline at end of file +.DS_Store diff --git a/data/employee_1.csv b/data/employee_1.csv index 88594c1..1c343a7 100644 --- a/data/employee_1.csv +++ b/data/employee_1.csv @@ -9,4 +9,4 @@ employee_id,week_nr,hours 333,30,10 444,15,2 444,16,4 -444,20,10 \ No newline at end of file +444,20,10 diff --git a/data/employee_2.csv b/data/employee_2.csv index 3cde3b2..4fd172b 100644 --- a/data/employee_2.csv +++ b/data/employee_2.csv @@ -9,4 +9,4 @@ employee_id,week_nr,hours 333,30,10 444,15,99 444,16,4 -444,20,99 \ No newline at end of file +444,20,99 diff --git a/data/employee_duplicate_keys_1.csv b/data/employee_duplicate_keys_1.csv index 485230f..da6aa13 100644 --- a/data/employee_duplicate_keys_1.csv +++ b/data/employee_duplicate_keys_1.csv @@ -9,4 +9,4 @@ employee_id,week_nr,hours 333,30,10 444,15,2 444,15,4 -444,20,10 \ No newline at end of file +444,20,10 diff --git a/data/employee_duplicate_keys_2.csv b/data/employee_duplicate_keys_2.csv index 87fe3b2..94e933c 100644 --- a/data/employee_duplicate_keys_2.csv +++ b/data/employee_duplicate_keys_2.csv @@ -9,4 +9,4 @@ employee_id,week_nr,hours 333,30,10 444,15,99 444,15,4 -444,20,99 \ No newline at end of file +444,20,99 diff --git a/data/sample_1.csv b/data/sample_1.csv index 92771da..05bf895 100644 --- a/data/sample_1.csv +++ b/data/sample_1.csv @@ -1,4 +1,4 @@ col_a,col_b,col_c 1,test,X 3,test,Z -4,test,A \ No newline at end of file +4,test,A diff --git a/data/sample_2.csv b/data/sample_2.csv index 9b9e509..61a1b7a 100644 --- a/data/sample_2.csv +++ b/data/sample_2.csv @@ -2,4 +2,4 @@ col_a,col_b,col_c 1,updated value,E 3,test,Z 5,new value,F -6,also new,H \ No newline at end of file +6,also new,H diff --git a/df_to_azure/settings.py b/df_to_azure/settings.py index 6a65146..766d067 100644 --- a/df_to_azure/settings.py +++ b/df_to_azure/settings.py @@ -1,6 +1,7 @@ from typing import Union from pandas import DataFrame + from df_to_azure.utils import test_unique_column_names diff --git a/df_to_azure/utils.py b/df_to_azure/utils.py index c587b30..b3eb871 100644 --- a/df_to_azure/utils.py +++ b/df_to_azure/utils.py @@ -2,7 +2,7 @@ import os import time -from df_to_azure.exceptions import PipelineRunError, DoubleColumnNamesError +from df_to_azure.exceptions import DoubleColumnNamesError, PipelineRunError def print_item(group): From 2f868944b8ca73e57869a817e578ce4e5ee7389f Mon Sep 17 00:00:00 2001 From: Melvin Folkers Date: Thu, 16 Mar 2023 09:14:43 +0100 Subject: [PATCH 08/17] changed old workflow setup to new (check with pre-commit) --- .github/workflows/ci.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b320a6e..3f9a8ef 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,16 +25,9 @@ jobs: run: | python -m pip install --upgrade pip pip install flake8 pytest black - - name: Lint with flake8 + - name: Run pre-commit run: | - # exit-zero treats all errors as warnings. - flake8 df_to_azure --count --ignore=E722 --max-complexity=10 --max-line-length=120 --statistics --per-file-ignores="__init__.py:F401" -# - name: Test with pytest -# run: | -# pytest df_to_azure - - name: Python Black Check - run: | - black --line-length=120 --check df_to_azure + pre-commit run --all-files - name: assert equality between setup.cfg and requirements.txt uses: actions/checkout@v2 - name: setup python From 53912c3bcacacf0172806e4e4df7311f55792ba5 Mon Sep 17 00:00:00 2001 From: Melvin Folkers Date: Thu, 16 Mar 2023 09:17:14 +0100 Subject: [PATCH 09/17] add install of requirement-dev.txt to workflow instead of hardcoded --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3f9a8ef..6d78d2c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest black + pip install -r requirements-dev.txt - name: Run pre-commit run: | pre-commit run --all-files From ac3106f79e83146a008c1b6edf79577feb4ce774 Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Thu, 16 Mar 2023 11:13:47 +0100 Subject: [PATCH 10/17] Add categorical mapping --- df_to_azure/__init__.py | 2 +- df_to_azure/export.py | 1 + setup.cfg | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/df_to_azure/__init__.py b/df_to_azure/__init__.py index 6b712d0..8eda3c1 100644 --- a/df_to_azure/__init__.py +++ b/df_to_azure/__init__.py @@ -2,7 +2,7 @@ from .export import df_to_azure -__version__ = "0.8.0" +__version__ = "0.8.1" logging.basicConfig( format="%(asctime)s.%(msecs)03d [%(levelname)-5s] [%(name)s] - %(message)s", diff --git a/df_to_azure/export.py b/df_to_azure/export.py index b76512b..2e39747 100644 --- a/df_to_azure/export.py +++ b/df_to_azure/export.py @@ -252,6 +252,7 @@ def column_types(self) -> dict: dtype("bool"): Boolean(), BooleanDtype(): Boolean(), DatetimeTZDtype(tz="utc"): DateTime(), + dtype("category"): string, } col_types = {col_name: type_conversion[col_type] for col_name, col_type in self.df.dtypes.to_dict().items()} diff --git a/setup.cfg b/setup.cfg index c0765ef..b8edf86 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = df_to_azure -version = 0.8.0 +version = 0.8.1 author = Melvin Folkers, Erfan Nariman author_email = melvin@zypp.io, erfan@zypp.io description = Automatically write pandas DataFrames to SQL by creating pipelines in Azure Data Factory with copy activity from blob to SQL From 59d1b35420fa27272026897905aaf41e872cb524 Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Thu, 16 Mar 2023 11:23:02 +0100 Subject: [PATCH 11/17] Fix pandas cat type --- df_to_azure/export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/df_to_azure/export.py b/df_to_azure/export.py index 2e39747..b54920c 100644 --- a/df_to_azure/export.py +++ b/df_to_azure/export.py @@ -10,6 +10,7 @@ from numpy import dtype from pandas import ( BooleanDtype, + CategoricalDtype, DataFrame, DatetimeTZDtype, Float64Dtype, @@ -252,7 +253,7 @@ def column_types(self) -> dict: dtype("bool"): Boolean(), BooleanDtype(): Boolean(), DatetimeTZDtype(tz="utc"): DateTime(), - dtype("category"): string, + CategoricalDtype(): string, } col_types = {col_name: type_conversion[col_type] for col_name, col_type in self.df.dtypes.to_dict().items()} From b8c52316f2cecd0daabe287af4a537695345bff3 Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Thu, 16 Mar 2023 15:21:23 +0100 Subject: [PATCH 12/17] Up package version --- requirements.txt | 6 +++--- setup.cfg | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index f9cfdbd..95006dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ azure-identity~=1.7.1 azure-mgmt-datafactory~=2.2.0 azure-mgmt-resource~=20.1.0 -azure-storage-blob~=12.8.1 -pandas~=1.4.1 -pyarrow~=7.0.0 +azure-storage-blob~=12.15.0 +pandas~=1.5.3 +pyarrow~=11.0.0 pyodbc~=4.0.32 sqlalchemy~=1.4.31 diff --git a/setup.cfg b/setup.cfg index b8edf86..5827838 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,9 +23,9 @@ install_requires = azure-identity~=1.7.1 azure-mgmt-datafactory~=2.2.0 azure-mgmt-resource~=20.1.0 - azure-storage-blob~=12.8.1 - pandas~=1.4.1 - pyarrow~=7.0.0 + azure-storage-blob~=12.15.0 + pandas~=1.5.3 + pyarrow~=11.0.0 pyodbc~=4.0.32 sqlalchemy~=1.4.31 From 40898c5d43988507ffeb9fcca5e1b47a4136143b Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 17 Mar 2023 08:23:44 +0100 Subject: [PATCH 13/17] Loosen requirements --- requirements.txt | 16 ++++++++-------- setup.cfg | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/requirements.txt b/requirements.txt index 95006dc..a2b7aa0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -azure-identity~=1.7.1 -azure-mgmt-datafactory~=2.2.0 -azure-mgmt-resource~=20.1.0 -azure-storage-blob~=12.15.0 -pandas~=1.5.3 -pyarrow~=11.0.0 -pyodbc~=4.0.32 -sqlalchemy~=1.4.31 +azure-identity>=1.7.1 +azure-mgmt-datafactory>=2.2.0 +azure-mgmt-resource>=20.1.0 +azure-storage-blob>=12.8.1 +pandas>=1.4.1 +pyarrow>=7.0.0 +pyodbc>=4.0.32 +sqlalchemy>=1.4.31 diff --git a/setup.cfg b/setup.cfg index 5827838..f4bdd8b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,14 +20,14 @@ classifiers = packages = df_to_azure python_requires = >=3.7 install_requires = - azure-identity~=1.7.1 - azure-mgmt-datafactory~=2.2.0 - azure-mgmt-resource~=20.1.0 - azure-storage-blob~=12.15.0 - pandas~=1.5.3 - pyarrow~=11.0.0 - pyodbc~=4.0.32 - sqlalchemy~=1.4.31 + azure-identity>=1.7.1 + azure-mgmt-datafactory>=2.2.0 + azure-mgmt-resource>=20.1.0 + azure-storage-blob>=12.8.1 + pandas>=1.4.1 + pyarrow>=7.0.0 + pyodbc>=4.0.32 + sqlalchemy>=1.4.31 [flake8] From 9003ed7dd7b483fa54a003a9bd499516d33046b4 Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 17 Mar 2023 15:07:10 +0100 Subject: [PATCH 14/17] Fix requirements --- requirements.txt | 4 ++-- setup.cfg | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index a2b7aa0..50c0a9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ azure-identity>=1.7.1 -azure-mgmt-datafactory>=2.2.0 +azure-mgmt-datafactory>=2.2.0,<2.7.0 azure-mgmt-resource>=20.1.0 azure-storage-blob>=12.8.1 pandas>=1.4.1 pyarrow>=7.0.0 pyodbc>=4.0.32 -sqlalchemy>=1.4.31 +sqlalchemy>=1.4.31,<2.0.0 diff --git a/setup.cfg b/setup.cfg index f4bdd8b..2cf4d73 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,13 +21,14 @@ packages = df_to_azure python_requires = >=3.7 install_requires = azure-identity>=1.7.1 - azure-mgmt-datafactory>=2.2.0 + azure-mgmt-datafactory>=2.2.0,<2.7.0 azure-mgmt-resource>=20.1.0 azure-storage-blob>=12.8.1 pandas>=1.4.1 pyarrow>=7.0.0 pyodbc>=4.0.32 - sqlalchemy>=1.4.31 + sqlalchemy>=1.4.31,<2.0.0 + [flake8] From 878b7a3c3c27ed0283dcd756d3d5ffb51ff41bcb Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 17 Mar 2023 20:02:55 +0100 Subject: [PATCH 15/17] Fix test and replace deprecated parameter --- df_to_azure/db.py | 4 ++-- df_to_azure/export.py | 2 +- df_to_azure/tests/test_general.py | 32 +++++++++---------------------- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/df_to_azure/db.py b/df_to_azure/db.py index 8abaddd..2c1e6b8 100644 --- a/df_to_azure/db.py +++ b/df_to_azure/db.py @@ -71,14 +71,14 @@ def create_stored_procedure(self): ) -def auth_azure(): +def auth_azure(driver: str = "ODBC Driver 17 for SQL Server"): connection_string = "mssql+pyodbc://{}:{}@{}:1433/{}?driver={}".format( os.environ.get("SQL_USER"), quote_plus(os.environ.get("SQL_PW")), os.environ.get("SQL_SERVER"), os.environ.get("SQL_DB"), - "ODBC Driver 17 for SQL Server", + driver, ) con = create_engine(connection_string).connect() diff --git a/df_to_azure/export.py b/df_to_azure/export.py index b54920c..e1af7a5 100644 --- a/df_to_azure/export.py +++ b/df_to_azure/export.py @@ -197,7 +197,7 @@ def upload_to_blob(self): blob=f"{self.table_name}/{self.table_name}.csv", ) - data = self.df.to_csv(index=False, sep="^", quotechar='"', line_terminator="\n") + data = self.df.to_csv(index=False, sep="^", quotechar='"', lineterminator="\n") blob_client.upload_blob(data, overwrite=True) def create_schema(self): diff --git a/df_to_azure/tests/test_general.py b/df_to_azure/tests/test_general.py index e67af4d..ac7df81 100644 --- a/df_to_azure/tests/test_general.py +++ b/df_to_azure/tests/test_general.py @@ -161,14 +161,15 @@ def test_pipeline_name(): def test_empty_dataframe(): df = DataFrame() - with pytest.raises(SystemExit): - df_to_azure( - df=df, - tablename="empty_dataframe", - schema="test", - method="create", - wait_till_finished=True, - ) + adf_client, run_response = df_to_azure( + df=df, + tablename="empty_dataframe", + schema="test", + method="create", + wait_till_finished=True, + ) + assert adf_client is None + assert run_response is None def test_convert_bigint(): @@ -203,18 +204,3 @@ def test_double_column_names(): schema="test", wait_till_finished=True, ) - - -def test_dataframe_with_no_data(): - - # scenario where there are 2 dataframes, first one empty and the second one with data. - df_list = [DataFrame(), DataFrame({"A": [1, 2, 3], "B": [10, 20, 30], "C": ["X", "Y", "Z"]})] - - results = list() - - for df in df_list: - r = df_to_azure(df, tablename=f"dataset_{df.shape[0]}_records", schema="test") - results.append(r) - - # there should be 2 results from the 2 df_to_azure operations. - assert len(results) == 2 From 179b4e5f77a4b1da896b33e04f2ee631ca933f4c Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 17 Mar 2023 20:05:03 +0100 Subject: [PATCH 16/17] Up version number --- df_to_azure/__init__.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/df_to_azure/__init__.py b/df_to_azure/__init__.py index 8eda3c1..cb22d33 100644 --- a/df_to_azure/__init__.py +++ b/df_to_azure/__init__.py @@ -2,7 +2,7 @@ from .export import df_to_azure -__version__ = "0.8.1" +__version__ = "0.9.0" logging.basicConfig( format="%(asctime)s.%(msecs)03d [%(levelname)-5s] [%(name)s] - %(message)s", diff --git a/setup.cfg b/setup.cfg index 2cf4d73..e863c66 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = df_to_azure -version = 0.8.1 +version = 0.9.0 author = Melvin Folkers, Erfan Nariman author_email = melvin@zypp.io, erfan@zypp.io description = Automatically write pandas DataFrames to SQL by creating pipelines in Azure Data Factory with copy activity from blob to SQL From 5fb1cdfb4636cc332675a5a7ab393df276a35419 Mon Sep 17 00:00:00 2001 From: Tim van der Heijden Date: Fri, 24 Mar 2023 08:34:08 +0100 Subject: [PATCH 17/17] Set pandas --- requirements.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 50c0a9b..6c846fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ azure-identity>=1.7.1 azure-mgmt-datafactory>=2.2.0,<2.7.0 azure-mgmt-resource>=20.1.0 azure-storage-blob>=12.8.1 -pandas>=1.4.1 +pandas>=1.5.0 pyarrow>=7.0.0 pyodbc>=4.0.32 sqlalchemy>=1.4.31,<2.0.0 diff --git a/setup.cfg b/setup.cfg index e863c66..5c71130 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ install_requires = azure-mgmt-datafactory>=2.2.0,<2.7.0 azure-mgmt-resource>=20.1.0 azure-storage-blob>=12.8.1 - pandas>=1.4.1 + pandas>=1.5.0 pyarrow>=7.0.0 pyodbc>=4.0.32 sqlalchemy>=1.4.31,<2.0.0