Merge branch 'dev' into de/sc-27472/arrow_nullable

.clang-format

@@ -64,6 +64,7 @@ IndentCaseLabels: true
 IndentRequiresClause: true
 IndentWidth:     2
 IndentWrappedFunctionNames: false
+InsertNewlineAtEOF: true
 JavaScriptQuotes: Leave
 JavaScriptWrapImports: true
 KeepEmptyLinesAtTheStartOfBlocks: false

.github/CODEOWNERS

@@ -0,0 +1,3 @@
+# Flag all PR configuration changes for documentation follow-up.
+/tiledb/sm/cpp_api/config.h   @TileDB-Inc/tiledb-docs
+/tiledb/api/c_api/config/config_api_external.h  @TileDB-Inc/tiledb-docs

.github/workflows/append-release-cmake.yml

@@ -0,0 +1,80 @@
+name: Append Release CMake
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Ref to be used as release'
+        default: 'latest'
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Ref to be used as release'
+        default: 'latest'
+        required: true
+        type: string
+
+jobs:
+  generate_cmake_files:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout TileDB
+        uses: actions/checkout@v3
+
+      - name: Make release and output directories
+        run: |
+          mkdir release output
+
+      - name: Github release data
+        id: release_data
+        uses: KevinRohn/github-full-release-data@v2.0.4
+        with:
+          # repository: 'TileDB-Inc/TileDB'
+          version: ${{ inputs.ref }}
+          asset-file: '*.zip,*.tar.gz'
+          asset-output: './release/'
+
+      - name: Render template
+        run: |
+          PATTERN="tiledb-([^-]+)-([^-]+)(-noavx2)?-(.+)\.(tar\.gz|zip)$"
+          RELLIST="output/releases.csv"
+          MODULE="output/DownloadPrebuiltTileDB.cmake"
+          cp cmake/inputs/DownloadPrebuiltTileDB.cmake $MODULE
+          echo "platform,url,sha256" > $RELLIST
+          
+          for FILE in $(ls release)
+          do
+                  if [[ $FILE =~ $PATTERN ]]
+                  then
+                          OS=${BASH_REMATCH[1]^^}
+                          ARCH=${BASH_REMATCH[2]^^}
+                          NOAVX2=${BASH_REMATCH[3]^^}
+                          PLATFORM=${OS}-${ARCH}${NOAVX2}
+          
+                          URL="${{ github.server_url }}/${{ github.repository }}/releases/download/${{ inputs.ref }}/$FILE"
+                          HASH=$(cat release/$FILE.sha256 | cut -d \t -f 1)
+          
+                          echo "${PLATFORM},${URL},${HASH}" >> $RELLIST
+                  fi
+          done
+          
+          SOURCE_FILE_NAME=$(ls release/tiledb-source-*.tar.gz)
+          URL_TILEDB_SOURCE="${{ github.server_url }}/${{ github.repository }}/releases/download/${{ inputs.ref }}/$(basename $SOURCE_FILE_NAME)"
+          HASH_TILEDB_SOURCE=$(cat $SOURCE_FILE_NAME.sha256 | cut -d \t -f 1)
+          
+          echo "source,${URL_TILEDB_SOURCE},${HASH_TILEDB_SOURCE}" >> $RELLIST
+          
+          HASH=$(sha256sum $RELLIST | cut -d " " -f 1)
+          echo $HASH > $RELLIST.sha256
+          
+          cat $RELLIST
+
+      - name: Upload template to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file: output/*
+          tag: ${{ steps.release_data.outputs.tag_name }}
+          overwrite: true
+          file_glob: true

.github/workflows/backport.yml

@@ -1,6 +1,6 @@
 name: Backport
 on:
-  pull_request:
+  pull_request_target:
     types:
       - closed
       - labeled
@@ -24,7 +24,7 @@ jobs:
       )
     steps:
       - name: Backport
-        uses: zephyrproject-rtos/action-backport@v2.0.3-3
+        uses: zephyrproject-rtos/action-backport@7e74f601d11eaca577742445e87775b5651a965f #tag=v2.0.3-3
         with:
           issue_labels: Backport
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-docs.yml

@@ -1,6 +1,9 @@
 name: build-docs
 on:
   push:
+    branches:
+      - dev
+      - 'release-*'
     paths-ignore:
       - '.github/workflows/quarto-render.yml'
       - '_quarto.yml'
@@ -18,13 +21,15 @@ on:
       - '**/.md'
       - 'tiledb/doxygen/source/*'
       - 'tiledb/sm/c_api/tiledb_version.h'
+  workflow_dispatch:
 
 jobs:
   build:
     runs-on: ubuntu-22.04
-    if: ${{ startsWith(github.ref , 'refs/tags') != true && startsWith(github.ref , 'build-') != true }}
     timeout-minutes: 90
     name: Build Docs
+    env:
+      VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
     steps:
       - uses: actions/checkout@v3
       - name: 'Print env'
@@ -38,23 +43,28 @@ jobs:
           printenv
         shell: bash
 
+      - name: Set environment variables for vcpkg binary caching
+        uses: actions/github-script@v6
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
       - name: Set up Python
         uses: actions/setup-python@v4
 
-      - name: Install Doxygen (linux only)
+      - name: Install Doxygen
         run: |
           set -e pipefail
           sudo apt-get update
           # Install doxygen *before* running cmake
           sudo apt-get install -y  doxygen
           pip install virtualenv
         shell: bash
-        if: ${{ runner.os == 'Linux' }}
 
-      - name: Build Doxygen Docs (linux only)'
+      - name: Build Doxygen Docs
         run: |
           # Build the documentation (this does not deploy to RTD).
           cd $GITHUB_WORKSPACE/tiledb/doxygen;
           ./local-build.sh;
         shell: bash
-        if: ${{ runner.os == 'Linux' }}

.github/workflows/build-rtools40.yml

@@ -17,6 +17,28 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 1
+      - name: Get vcpkg commit to checkout
+        id: vcpkg_commit
+        shell: bash
+        run: |
+          echo "ref=$(grep -oP '(?<=set\(VCPKG_COMMIT_STRING )\w*' cmake/Options/TileDBToolchain.cmake | head -n 1)" >> "$GITHUB_OUTPUT"
+      # We clone vcpkg ourselves because having FetchContent do it inside
+      # the build directory causes compilation errors due to long paths.
+      - name: Checkout vcpkg repository
+        uses: actions/checkout@v3
+        with:
+          repository: microsoft/vcpkg
+          path: vcpkg
+          ref: ${{ steps.vcpkg_commit.outputs.ref }}
+          # Vcpkg requires fetching all commits.
+          fetch-depth: 0
+      # Configure required environment variables for vcpkg to use
+      # GitHub's Action Cache
+      - uses: actions/github-script@v6
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
       - name: Install rtools40 if needed
         uses: r-windows/install-rtools@master
       - name: Building TileDB with Rtools40
@@ -27,6 +49,8 @@ jobs:
         env:
           TILEDB_HOME: ${{ github.workspace }}
           MINGW_ARCH: ${{ matrix.msystem }}
+          VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
+          VCPKG_ROOT: ${{ github.workspace }}/vcpkg
         shell: c:\rtools40\usr\bin\bash.exe --login {0}
       - name: "Upload binaries"
         uses: actions/upload-artifact@v3

.github/workflows/build-ubuntu20.04-backwards-compatibility.yml

@@ -26,7 +26,7 @@ jobs:
       # Need this for virtualenv and arrow tests if enabled
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
       - run: |
           set -e pipefail
           python -m pip install --upgrade pip virtualenv
@@ -59,7 +59,7 @@ jobs:
           - ubuntu-20.04
         # Note: v2_1_0 arrays were never created so its currently skipped
         # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION
-        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_0", "v2_13_0", "v2_14_0", "v2_15_0", "v2_16_0"]
+        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0", "v2_24_0"]
     timeout-minutes: 30
     name: ${{ matrix.tiledb_version }}
     steps:
@@ -74,6 +74,16 @@ jobs:
       - name: Update tiledb_unit permissions
         run: chmod +x $GITHUB_WORKSPACE/build/tiledb/test/tiledb_unit
 
+      - name: Free disk space
+        run: |
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt clean
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: 'Test backward compatibility'
         id: test
         env: