Support-Sqlalchemy 1.4 part-1

* Update sqlalchemy dependency to base version to 1.4.0 * Update ssl/tls test to comply with new sqla 1.4 api * Remove backwards compatibility with SQLA < 1.4 for ssl/tls test * Disable statement cache for pyodbc and turbodbc dialects For more details see also: * #190 * https://docs.sqlalchemy.org/en/14/core/connections.html#engine-thirdparty-caching * https://docs.sqlalchemy.org/en/14/faq/performance.html#why-is-my-application-slow-after-upgrading-to-1-4-and-or-2-x * Upgrade tests to use new introspection api * Update URL creation to new api * Fix pre_exec edge case * Refactor custom merge statement for better readability * Fix translate map regression test * Update dev dependencies * Disable unsupported features regarding binary types * Restructure intigration test setup, to mitigate side effects * Update mypy config * Bump poetry versions in github workflows
exasol · Nov 30, 2022 · b0044d3 · b0044d3
1 parent 318ef14
commit b0044d3
Show file tree

Hide file tree

Showing 29 changed files with 1,019 additions and 663 deletions.
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -9,9 +9,9 @@ on:
 
 jobs:
 
-  build_docs:
+  docs:
     runs-on: ubuntu-latest
-    name: build_docs
+    name: Build Documentation
 
     steps:
       - name: Checkout
@@ -32,9 +32,42 @@ jobs:
         run: |
           poetry run python -m nox -s build-docs
 
-  run_tests:
+  checks:
     runs-on: ubuntu-latest
-    needs: build_docs
+    name: Project Checks (Python-${{ matrix.python }})
+    strategy:
+      fail-fast: false
+      matrix:
+        python:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+    steps:
+      - name: Fetch sqlalchemy_exasol code from repository
+        uses: actions/checkout@v3
+
+      - name: Install via apt
+        run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev
+
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2.1.5
+        with:
+          poetry-version: 1.2.1
+
+      - name: Install python project dependencies
+        run: poetry install
+
+      - name: Project Checks (Python-${{ matrix.python }})
+        run: poetry run nox -s check
+
+  tests:
+    runs-on: ubuntu-latest
+    needs: [docs, checks]
     strategy:
       fail-fast: false
       matrix:
@@ -49,79 +82,74 @@ jobs:
           - 7.1.9
           - 7.0.18
 
-    name: Run Tests (Python-${{ matrix.python }}, Connector-${{ matrix.connector }}, Exasol-${{ matrix.exasol_version }})
+    name: Integration Tests (Python-${{ matrix.python }}, Connector-${{ matrix.connector }}, Exasol-${{ matrix.exasol_version }})
 
     steps:
 
-    - name: Fetch sqlalchemy_exasol code from repository
-      uses: actions/checkout@v3
-
-    - name: Setup integration-test-docker-environment
-      uses: actions/setup-python@v4
+      - name: Fetch sqlalchemy_exasol code from repository
+        uses: actions/checkout@v3
 
-    - name: Install via apt
-      run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev
+      - name: Install via apt
+        run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev
 
-    - name: Setup Python ${{ matrix.python }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python }}
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
 
-    - name: Install poetry
-      uses: abatilo/actions-poetry@v2.1.5
-      with:
-        poetry-version: 1.1.13
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2.1.5
+        with:
+          poetry-version: 1.2.1
 
-    - name: Install python project dependencies
-      run: poetry install
+      - name: Install python project dependencies
+        run: poetry install
 
-    - name: Install python project dependencies including trubodbc
-      run: poetry install --extras "turbodbc"
-      if: ${{ matrix.connector == 'turbodbc' }}
+      - name: Install python project dependencies including trubodbc
+        run: poetry install --extras "turbodbc"
+        if: ${{ matrix.connector == 'turbodbc' }}
 
-    - name: Checkout test environment
-      run: git clone --depth 1 --branch ${ITDE_TAG} ${ITDE_URL}
-      working-directory: ..
-      env:
-        ITDE_URL: "https://github.com/exasol/integration-test-docker-environment.git"
-        ITDE_TAG: "0.11.0"
+      - name: Checkout test environment
+        run: git clone --depth 1 --branch ${ITDE_TAG} ${ITDE_URL}
+        working-directory: ..
+        env:
+          ITDE_URL: "https://github.com/exasol/integration-test-docker-environment.git"
+          ITDE_TAG: "0.11.0"
 
-    - name: Run Test for Python ${{ matrix.python }} using ${{ matrix.connector }}
-      run: poetry run nox -s "verify(connector='${{ matrix.connector }}', db_version='${{ matrix.exasol_version }}')"
+      - name: Run Test for Python ${{ matrix.python }} using ${{ matrix.connector }}
+        run: poetry run nox -s integration-tests -- --connector ${{ matrix.connector }} --db-version ${{ matrix.exasol_version }}
 
   upload_to_pypi:
     runs-on: ubuntu-latest
-    needs: run_tests
+    needs: tests
+    name: Build & Upload Package [PYPI]
     if: startsWith(github.event.ref, 'refs/tags')
     strategy:
       matrix:
-        python: [3.8]
-
-    name: Build & Upload Package [PYPI]
+        python: [ 3.8 ]
 
     steps:
+      - name: Fetch sqlalchemy_exasol code from repository
+        uses: actions/checkout@v3
 
-    - name: Fetch sqlalchemy_exasol code from repository
-      uses: actions/checkout@v3
-
-    - name: Fetch all tags
-      run: git fetch origin +refs/tags/*:refs/tags/*
+      - name: Fetch all tags
+        run: git fetch origin +refs/tags/*:refs/tags/*
 
-    - name: Setup Python ${{ matrix.python }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python }}
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
 
-    - name: Install poetry
-      uses: abatilo/actions-poetry@v2.1.5
-      with:
-        poetry-version: 1.1.13
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2.1.5
+        with:
+          poetry-version: 1.2.1
 
-    - name: Install python project dependencies
-      run: poetry install
+      - name: Install python project dependencies
+        run: poetry install
 
-    - name: Build and push package to PYPI
-      env:
-        POETRY_HTTP_BASIC_PYPI_USERNAME: "__token__"
-        POETRY_HTTP_BASIC_PYPI_PASSWORD: "${{ secrets.pypi_token }}"
-      run: poetry run nox -s release
+      - name: Build and push package to PYPI
+        env:
+          POETRY_HTTP_BASIC_PYPI_USERNAME: "__token__"
+          POETRY_HTTP_BASIC_PYPI_PASSWORD: "${{ secrets.pypi_token }}"
+        run: poetry run nox -s release
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -13,9 +13,9 @@ on:
 
 jobs:
 
-  build_docs:
+  docs:
     runs-on: ubuntu-latest
-    name: build_docs
+    name: Build Documentation
 
     steps:
       - name: Checkout
@@ -36,9 +36,42 @@ jobs:
         run: |
           poetry run python -m nox -s build-docs
 
-  run_tests:
+  checks:
     runs-on: ubuntu-latest
-    needs: build_docs
+    name: Project Checks (Python-${{ matrix.python }})
+    strategy:
+      fail-fast: false
+      matrix:
+        python:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+    steps:
+      - name: Fetch sqlalchemy_exasol code from repository
+        uses: actions/checkout@v3
+
+      - name: Install via apt
+        run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev
+
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2.1.5
+        with:
+          poetry-version: 1.2.1
+
+      - name: Install python project dependencies
+        run: poetry install
+
+      - name: Project Checks (Python-${{ matrix.python }})
+        run: poetry run nox -s check
+
+  tests:
+    runs-on: ubuntu-latest
+    needs: [docs, checks]
     strategy:
       fail-fast: false
       matrix:
@@ -53,72 +86,71 @@ jobs:
           - 7.1.9
           - 7.0.18
 
-    name: Run Tests (Python-${{ matrix.python }}, Connector-${{ matrix.connector }}, Exasol-${{ matrix.exasol_version }})
+    name: Integration Tests (Python-${{ matrix.python }}, Connector-${{ matrix.connector }}, Exasol-${{ matrix.exasol_version }})
 
     steps:
 
-    - name: Fetch sqlalchemy_exasol code from repository
-      uses: actions/checkout@v3
+      - name: Fetch sqlalchemy_exasol code from repository
+        uses: actions/checkout@v3
 
-    - name: Install via apt
-      run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev
+      - name: Install via apt
+        run: sudo apt-get install unixodbc unixodbc-dev libboost-date-time-dev libboost-locale-dev libboost-system-dev
 
-    - name: Setup Python ${{ matrix.python }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python }}
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
 
-    - name: Install poetry
-      uses: abatilo/actions-poetry@v2.1.5
-      with:
-        poetry-version: 1.1.13
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2.1.5
+        with:
+          poetry-version: 1.2.1
 
-    - name: Install python project dependencies
-      run: poetry install
+      - name: Install python project dependencies
+        run: poetry install
 
-    - name: Install python project dependencies including trubodbc
-      run: poetry install --extras "turbodbc"
-      if: ${{ matrix.connector == 'turbodbc' }}
+      - name: Install python project dependencies including trubodbc
+        run: poetry install --extras "turbodbc"
+        if: ${{ matrix.connector == 'turbodbc' }}
 
-    - name: Checkout test environment
-      run: git clone --depth 1 --branch ${ITDE_TAG} ${ITDE_URL}
-      working-directory: ..
-      env:
-        ITDE_URL: "https://github.com/exasol/integration-test-docker-environment.git"
-        ITDE_TAG: "0.11.0"
+      - name: Checkout test environment
+        run: git clone --depth 1 --branch ${ITDE_TAG} ${ITDE_URL}
+        working-directory: ..
+        env:
+          ITDE_URL: "https://github.com/exasol/integration-test-docker-environment.git"
+          ITDE_TAG: "0.11.0"
 
-    - name: Run Test for Python ${{ matrix.python }} using ${{ matrix.connector }}
-      run: poetry run nox -s "verify(connector='${{ matrix.connector }}', db_version='${{ matrix.exasol_version }}')"
+      - name: Run Test for Python ${{ matrix.python }} using ${{ matrix.connector }}
+        run: poetry run nox -s integration-tests -- --connector ${{ matrix.connector }} --db-version ${{ matrix.exasol_version }}
 
   build_package:
     runs-on: ubuntu-latest
-    needs: run_tests
+    name: Build Package
+    needs: tests
     strategy:
       matrix:
-        python: [3.8]
-
-    name: Build Package
+        python: [ 3.8 ]
 
     steps:
 
-    - name: Fetch sqlalchemy_exasol code from repository
-      uses: actions/checkout@v3
+      - name: Fetch sqlalchemy_exasol code from repository
+        uses: actions/checkout@v3
 
-    - name: Fetch all tags
-      run: git fetch origin +refs/tags/*:refs/tags/*
+      - name: Fetch all tags
+        run: git fetch origin +refs/tags/*:refs/tags/*
 
-    - name: Setup Python ${{ matrix.python }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python }}
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
 
-    - name: Install poetry
-      uses: abatilo/actions-poetry@v2.1.5
-      with:
-        poetry-version: 1.1.13
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2.1.5
+        with:
+          poetry-version: 1.1.13
 
-    - name: Install python project dependencies
-      run: poetry install
+      - name: Install python project dependencies
+        run: poetry install
 
-    - name: Build sdist and wheel packages
-      run: poetry build
+      - name: Build sdist and wheel packages
+        run: poetry build
diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml
@@ -23,7 +23,7 @@ jobs:
       - name: Install Dependencies
         run: |
           poetry install
-      - name: Build Documentations
+      - name: Build Documentation
         run: |
           poetry run python -m nox -s build-docs
           touch doc/build/.nojekyll

diff --git a/.github/workflows/link-check.yaml b/.github/workflows/link-check.yaml
@@ -31,10 +31,10 @@ jobs:
       - name: Install poetry
         uses: abatilo/actions-poetry@v2.1.5
         with:
-          poetry-version: 1.1.13
+          poetry-version: 1.2.1
 
       - name: Install python project dependencies
         run: poetry install --no-root
 
       - name: Check documentation links ${{ matrix.python }} using ${{ matrix.connector }}
-        run: poetry run nox -s "check-links"
+        run: poetry run nox -s check-links
diff --git a/changelog.d/20220810_120055_nicola.coretti__106_Support_Sqlalchemy_1_4.rst b/changelog.d/20220810_120055_nicola.coretti__106_Support_Sqlalchemy_1_4.rst
@@ -0,0 +1,10 @@
+🔧 Changed
+----------
+- Update SQLAlchemy dependency to >= 1.4
+
+    .. warning::
+
+        This may impact the performance (see also `SQLAlchemy docs <https://docs.sqlalchemy.org/en/14/faq/performance.html#why-is-my-application-slow-after-upgrading-to-1-4-and-or-2-x>`_).
+        If you are not willing or able to pay those potentional performance hits, you should wait until the `tracking-issue <https://github.com/exasol/sqlalchemy-exasol/issues/190>`_
+        is resolved.
+