Merge branch 'dev' into de/sc-27472/arrow_nullable

TileDB-Inc · Aug 5, 2024 · 6a13882 · 6a13882
2 parents 339bebe + 2a30fbe
commit 6a13882
Show file tree

Hide file tree

Showing 408 changed files with 12,725 additions and 12,303 deletions.
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -1,30 +1,26 @@
 name: Backport
 on:
-  pull_request_target:
-    types:
-      - closed
-      - labeled
-    paths-ignore:
-      - '_quarto.yml'
-      - 'quarto-materials/*'
-      - '**/.md'
+  issue_comment:
+    types: [created]
+  schedule:
+    # once a day at 13:00 UTC to cleanup old runs
+    - cron: '0 13 * * *'
+
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+  actions: write
 
 jobs:
   backport:
-    name: Backport
-    runs-on: ubuntu-22.04
-    if: >
-      github.event.pull_request.merged &&
-      (
-        github.event.action == 'closed' ||
-        (
-          github.event.action == 'labeled' &&
-          contains(github.event.label.name, 'backport')
-        )
-      )
-    steps:
-      - name: Backport
-        uses: zephyrproject-rtos/action-backport@7e74f601d11eaca577742445e87775b5651a965f #tag=v2.0.3-3
-        with:
-          issue_labels: Backport
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+    uses: dotnet/arcade/.github/workflows/backport-base.yml@aa70f3a7a494063952307a9b1df7ca0ab67bce13
+    with:
+      pr_title_template: '[Backport %target_branch%] %source_pr_title% (#%source_pr_number%)'
+      pr_description_template: |
+        Backport of #%source_pr_number% to %target_branch%
+
+        ---
+        TYPE: NO_HISTORY | FEATURE | BUG | IMPROVEMENT | DEPRECATION | C_API | CPP_API | BREAKING_BEHAVIOR | BREAKING_API | FORMAT
+        DESC: <short description>
+      repository_owners: TileDB-Inc
diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml
@@ -46,7 +46,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: tiledb_unit
-          path: ${{ github.workspace }}/build/tiledb/test/tiledb_unit
+          path: ${{ github.workspace }}/build/test/tiledb_unit
           retention-days: 1
 
   test:
@@ -69,10 +69,10 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: tiledb_unit
-          path: ${{ github.workspace }}/build/tiledb/test/
+          path: ${{ github.workspace }}/build/test/
 
       - name: Update tiledb_unit permissions
-        run: chmod +x $GITHUB_WORKSPACE/build/tiledb/test/tiledb_unit
+        run: chmod +x $GITHUB_WORKSPACE/build/test/tiledb_unit
 
       - name: Free disk space
         run: |
@@ -106,7 +106,7 @@ jobs:
 
 
           # Bypass Catch2 Framework stdout interception with awk on test output
-          $GITHUB_WORKSPACE/build/tiledb/test/tiledb_unit -d yes "[backwards-compat]"| awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          $GITHUB_WORKSPACE/build/test/tiledb_unit -d yes "[backwards-compat]"| awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
       - name: 'Test status check'
         run: |

diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml
@@ -242,11 +242,11 @@ jobs:
             # & "$env:BUILD_SOURCESDIRECTORY\scripts\install-minio.ps1"
 
             # update CMake to disable S3 for the test configuration, see minio note above
-            cmake -B $env:BUILD_BUILDDIRECTORY\tiledb -DTILEDB_S3=0 $env:BUILD_SOURCESDIRECTORY
+            cmake -B $env:BUILD_BUILDDIRECTORY -DTILEDB_S3=OFF $env:BUILD_SOURCESDIRECTORY
           }
 
           # CMake exits with non-0 status if there are any warnings during the build, so
-          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit unit_vfs tiledb_regression all_link_complete --config $CMakeBuildType
+          cmake --build $env:BUILD_BUILDDIRECTORY -j --target tiledb_unit unit_vfs tiledb_regression all_link_complete --config $CMakeBuildType
 
           if ($env:TILEDB_AZURE -eq "ON") {
             if($env.TILEDB_USE_CUSTOM_NODE_JS) {
@@ -277,23 +277,23 @@ jobs:
 
           # Actually run tests
 
-          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\tiledb_unit.exe -d=yes"
+          $cmds = "$env:BUILD_BUILDDIRECTORY\test\tiledb_unit.exe -d=yes"
           Write-Host "cmds: '$cmds'"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
              Write-Host "Tests failed. tiledb_unit exit status: " $LastExitCocde
              $host.SetShouldExit($LastExitCode)
           }
           
-          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\filesystem\test\unit_vfs -d=yes"
+          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\sm\filesystem\test\unit_vfs -d=yes"
           Write-Host "cmds: '$cmds'"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
              Write-Host "Tests failed. tiledb_vfs exit status: " $LastExitCocde
              $host.SetShouldExit($LastExitCode)
           }
 
-          $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\test_assert.exe -d=yes"
+          $cmds = "$env:BUILD_BUILDDIRECTORY\test\ci\test_assert.exe -d=yes"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
              Write-Host "Tests failed. test_assert exit status: " $LastExitCocde
@@ -313,9 +313,9 @@ jobs:
 
           $env:Path += ";$env:BUILD_BUILDDIRECTORY\dist\bin;$env:BUILD_BUILDDIRECTORY\externals\install\bin"
 
-          $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api")
-          $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\test_app_data")
-          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\") -Filter *.exe |
+          $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "examples\c_api")
+          $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "examples\c_api\test_app_data")
+          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "examples\c_api\") -Filter *.exe |
           Foreach-Object {
             try {
               Set-Location -path $TestAppDir
@@ -346,9 +346,9 @@ jobs:
           Set-Location -path $TestAppDir
           Remove-Item -Path $TestAppDataDir -recurse -force -ErrorAction SilentlyContinue
 
-          $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api")
-          $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\test_app_data")
-          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\") -Filter *.exe |
+          $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "examples\cpp_api")
+          $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "examples\cpp_api\test_app_data")
+          Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "examples\cpp_api\") -Filter *.exe |
           Foreach-Object {
             try {
               Set-Location -path $TestAppDir
@@ -405,7 +405,7 @@ jobs:
       - name: Packaging test
         shell: pwsh
         run: |
-          cmake --build $env:BUILD_BUILDDIRECTORY\tiledb --target check-package
+          cmake --build $env:BUILD_BUILDDIRECTORY --target check-package
 
       - name: 'process crashdumps'
         shell: cmd

diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml
@@ -82,6 +82,9 @@ env:
   bootstrap_args: "--enable-ccache --vcpkg-base-triplet=${{ inputs.vcpkg_base_triplet || (startsWith(inputs.matrix_image, 'ubuntu-') && 'x64-linux' || 'x64-osx') }} ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}"
   VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
   SCCACHE_GHA_ENABLED: "true"
+  # Manylinux does not support Node 20 due to libc incompatibility. Temporarily opt out.
+  # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: ${{ inputs.manylinux && 'true' || 'false' }}
 
 jobs:
   build:
@@ -124,7 +127,7 @@ jobs:
         if: inputs.manylinux
         run: |
           set -e pipefail
-          yum install -y redhat-lsb-core centos-release-scl devtoolset-7
+          yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd
           echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc
 
       # Need this for virtualenv and arrow tests if enabled
@@ -210,10 +213,10 @@ jobs:
           # Run tests
 
           # Bypass Catch2 Framework stdout interception with awk on test output
-          ./tiledb/test/regression/tiledb_regression -d yes
-          ./tiledb/test/ci/test_assert -d yes
-          ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
-          ./tiledb/tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./test/regression/tiledb_regression -d yes
+          ./test/ci/test_assert -d yes
+          ./test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
           
           ctest -R tiledb_timing_unit | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
@@ -241,7 +244,7 @@ jobs:
           ###################################################
           # Perform package test
 
-          cmake --build $GITHUB_WORKSPACE/build/tiledb --target check-package
+          cmake --build $GITHUB_WORKSPACE/build --target check-package
 
       - name: 'Dump core stacks on failure'
         if: ${{ failure() && startsWith(matrix.os, 'ubuntu-') == true }} # only run this job if the build step failed

diff --git a/.github/workflows/mingw-w64-tiledb/PKGBUILD b/.github/workflows/mingw-w64-tiledb/PKGBUILD
@@ -34,24 +34,22 @@ build() {
     -DTILEDB_TESTS=OFF \
     -DTILEDB_S3=ON \
     -DCOMPILER_SUPPORTS_AVX2=OFF \
-    -DTILEDB_SKIP_S3AWSSDK_DIR_LENGTH_CHECK=ON \
     -DTILEDB_WERROR=OFF \
     -DVCPKG_TARGET_TRIPLET=${vcpkg_triplet} \
     ..
   make
-  make -C tiledb
 
 # in a local environ, can build to demo/check for previously seen (windows event) handle leakge
 # apparently due to issues with mingw std library implementation.
 # note that with S3 enabled above, and minio not running, the aws sdk may fatally error on exit.
-#  make -C tiledb/test tiledb_unit # need the support lib built
-#  make -C tiledb/test/performance tiledb_explore_msys_handle_leakage
+#  make -C test tiledb_unit # need the support lib built
+#  make -C test/performance tiledb_explore_msys_handle_leakage
 #  . ../test/performance/msys_handle_leakage/trials.sh
 }
 
 package() {
   cd ${source_dir}/build-${MINGW_CHOST}
-  make DESTDIR="${pkgdir}" -C tiledb install
+  make DESTDIR="${pkgdir}" install
   # Copy vcpkg_installed to the mingw package.
   cp -r vcpkg_installed/${vcpkg_triplet}/. ${pkgdir}/${MINGW_ARCH}
 }

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -65,10 +65,13 @@ jobs:
             MACOSX_DEPLOYMENT_TARGET: 11
             triplet: arm64-osx-release
     runs-on: ${{ matrix.os }}
-    container: ${{ matrix.manylinux && 'quay.io/pypa/manylinux2014_x86_64:2023-11-13-f6b0c51' || '' }}
+    container: ${{ matrix.manylinux && 'quay.io/pypa/manylinux2014_x86_64' || '' }}
     env:
       MACOSX_DEPLOYMENT_TARGET: ${{ matrix.MACOSX_DEPLOYMENT_TARGET }}
       VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
+      # Manylinux does not support Node 20 due to libc incompatibility. Temporarily opt out.
+      # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: ${{ matrix.manylinux && 'true' || 'false' }}
 
     steps:
       - name: Checkout TileDB
@@ -94,7 +97,7 @@ jobs:
         if: matrix.manylinux
         run: |
           set -e pipefail
-          yum install -y redhat-lsb-core centos-release-scl devtoolset-7
+          yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd
           echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc
       - name: Configure TileDB
         run: |
@@ -122,8 +125,8 @@ jobs:
         with:
           name: release
           path: |
-            build/tiledb/tiledb-*.tar.gz*
-            build/tiledb/tiledb-*.zip*
+            build/tiledb-*.tar.gz*
+            build/tiledb-*.zip*
       - name: "Print log files (failed build only)"
         run: |
           source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh
@@ -198,9 +201,12 @@ jobs:
     needs: Publish-Release
     if: (failure() || cancelled()) && github.event_name != 'workflow_dispatch'
     steps:
+      # The open-issue action requires to clone the repository.
+      - name: Checkout TileDB
+        uses: actions/checkout@v4
       - name: Create Issue if Build Fails
         uses: TileDB-Inc/github-actions/open-issue@main
         with:
           name: Release failed
           label: bug
-          assignee: KiterLuc,teo-tsirpanis,davisp
+          assignee: KiterLuc,teo-tsirpanis,davisp
diff --git a/.github/workflows/test-cloud-e2e.yml b/.github/workflows/test-cloud-e2e.yml
@@ -90,7 +90,7 @@ jobs:
           # Run tests
 
           # Bypass Catch2 Framework stdout interception with awk on test output
-          ./tiledb/test/tiledb_unit -d yes "[vfs-e2e]" | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
+          ./test/tiledb_unit -d yes "[vfs-e2e]" | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
       - name: 'Test status check'
         run: |

diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml
@@ -61,14 +61,13 @@ jobs:
             -DCMAKE_BUILD_TYPE=Debug \
             -DTILEDB_SERIALIZATION=ON \
             -DTILEDB_ASSERTIONS=ON
-          cmake --build build -j4
           # Build all unit tests
-          cmake --build build/tiledb --target tests -j4
+          cmake --build build --target tests -j4
 
       - name: 'Run standalone unit tests'
         run: |
-          ctest --test-dir build/tiledb -R '(^unit_|test_assert)' --no-tests=error
-          ctest --test-dir build/tiledb -R 'test_ci_asserts'
+          ctest --test-dir build -R '(^unit_|test_assert)' --no-tests=error
+          ctest --test-dir build -R 'test_ci_asserts'
 
       - name: "Print log files (failed build only)"
         run: |