Merge branch 'dev' into de/sc-27472/arrow_nullable

.github/workflows/build-ubuntu20.04-backwards-compatibility.yml

@@ -59,7 +59,7 @@ jobs:
           - ubuntu-20.04
         # Note: v2_1_0 arrays were never created so its currently skipped
         # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION
-        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0", "v2_24_0"]
+        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0", "v2_24_0", "v2_25_0"]
     timeout-minutes: 30
     name: ${{ matrix.tiledb_version }}
     steps:

.github/workflows/build-windows.yml

@@ -24,27 +24,23 @@ jobs:
         include:
           - environ: 'azure'
             TILEDB_AZURE: ON
-            TILEDB_ARROW_TESTS: ON
             TILEDB_SERIALIZATION: OFF
             TILEDB_S3: OFF
             TILEDB_WEBP: OFF
           - environ: 's3'
             TILEDB_S3: ON
-            TILEDB_ARROW_TESTS: ON
             TILEDB_SERIALIZATION: OFF
             TILEDB_AZURE: OFF
             TILEDB_WEBP: OFF
           - environ: 'gcs'
             TILEDB_GCS: ON
-            TILEDB_ARROW_TESTS: ON
             TILEDB_SERIALIZATION: OFF
             TILEDB_AZURE: OFF
             TILEDB_S3: OFF
             TILEDB_WEBP: OFF
           - environ: 'serialization'
             TILEDB_S3: OFF
             TILEDB_AZURE: OFF
-            TILEDB_ARROW_TESTS: OFF
             TILEDB_SERIALIZATION: ON
             TILEDB_WEBP: ON
 
@@ -57,7 +53,6 @@ jobs:
       TILEDB_AZURE: ${{ matrix.TILEDB_AZURE }} #azure }}
       TILEDB_GCS: ${{ matrix.TILEDB_GCS }} #gcs }}
       TILEDB_SERIALIZATION: ${{ matrix.TILEDB_SERIALIZATION }} #serialization }}
-      TILEDB_ARROW_TESTS: ${{ matrix.TILEDB_ARROW_TESTS }}
       TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }}
       TILEDB_CMAKE_BUILD_TYPE: 'Release'
       VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
@@ -110,15 +105,6 @@ jobs:
           #but, it seems *we* can populate our own var from our actions, and access it in file upload...
           echo TDBLOCALAPPDATA=$env:LOCALAPPDATA >> "$env:GITHUB_ENV"
 
-      - name: ARROW python needs
-        shell: bash
-        if: ${{ matrix.TILEDB_ARROW_TESTS == 'ON' }}
-        run: |
-          set -e pipefail
-          if [[ "$TILEDB_ARROW_TESTS" == "ON" ]]; then
-            pip install pyarrow pybind11 numpy
-          fi
-
       - name: Prepare git
         run: git config --global core.autocrlf false
 
@@ -167,6 +153,9 @@ jobs:
           if ($env:TILEDB_TOOLS -eq "ON") {
             $bootstrapOptions = $bootstrapOptions + " -EnableTools"
           }
+          if ($env:TILEDB_ARROW_TESTS -eq "ON") {
+            $bootstrapOptions = $bootstrapOptions + " -EnableArrowTests"
+          }
           $CMakeBuildType = $env:TILEDB_CMAKE_BUILD_TYPE
           if ($env:TILEDB_DEBUG -eq "ON") {
             $bootstrapOptions = $bootstrapOptions + " -EnableDebug"
@@ -183,10 +172,6 @@ jobs:
           # if ($env:TILEDB_CI_TSAN -eq "ON") {
           #  $bootstrapOptions = $bootstrapOptions + " -EnableSanitizer thread -EnableDebug"
           # }
-          # static already added above as initial default
-          # if ($env:TILEDB_FORCE_BUILD_DEPS" -eq "ON") {
-          #   $bootstrapOptions = $bootstrapOptions + " -EnableBuildDeps"
-          # }
           if ($env:TILEDB_WERROR -eq "OFF") {
             $bootstrapOptions = $bootstrapOptions + " -DisableWerror"
           }
@@ -256,7 +241,7 @@ jobs:
             }
             else { #using the node/npm already present in vm
               Write-Host "azurite: using vm install nodejs"
-               #this code path avoids overhead of download/expand/install of alternate nodejs/azurite.
+              #this code path avoids overhead of download/expand/install of alternate nodejs/azurite.
               npm install -g azurite
               Write-Host "done with 'npm install -g azurite'"
               $azuriteDataPath = (Join-Path $env:TEMP "azuriteData")
@@ -281,23 +266,23 @@ jobs:
           Write-Host "cmds: '$cmds'"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
-             Write-Host "Tests failed. tiledb_unit exit status: " $LastExitCocde
-             $host.SetShouldExit($LastExitCode)
+              Write-Host "Tests failed. tiledb_unit exit status: " $LastExitCocde
+              $host.SetShouldExit($LastExitCode)
           }
-          
+
           $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\sm\filesystem\test\unit_vfs -d=yes"
           Write-Host "cmds: '$cmds'"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
-             Write-Host "Tests failed. tiledb_vfs exit status: " $LastExitCocde
-             $host.SetShouldExit($LastExitCode)
+              Write-Host "Tests failed. tiledb_vfs exit status: " $LastExitCocde
+              $host.SetShouldExit($LastExitCode)
           }
 
           $cmds = "$env:BUILD_BUILDDIRECTORY\test\ci\test_assert.exe -d=yes"
           Invoke-Expression $cmds
           if ($LastExitCode -ne 0) {
-             Write-Host "Tests failed. test_assert exit status: " $LastExitCocde
-             $host.SetShouldExit($LastExitCode)
+            Write-Host "Tests failed. test_assert exit status: " $LastExitCocde
+            $host.SetShouldExit($LastExitCode)
           }
       - name: Build examples
         shell: pwsh

.github/workflows/ci-linux_mac.yml

@@ -72,8 +72,6 @@ env:
   CMAKE_GENERATOR: ${{ inputs.cmake_generator }}
   TILEDB_CI_BACKEND: ${{ inputs.ci_backend }}
   TILEDB_CI_OS: ${{ startsWith(inputs.matrix_image, 'ubuntu-') && 'Linux' || 'macOS' }}
-  # Installing Python does not work on manylinux.
-  TILEDB_ARROW_TESTS: ${{ !inputs.manylinux && 'ON' || 'OFF' }}
   TILEDB_MANYLINUX: ${{ !inputs.manylinux && 'ON' || 'OFF' }}
   CXX: ${{ inputs.matrix_compiler_cxx }}
   CC: ${{ inputs.matrix_compiler_cc }}
@@ -130,26 +128,11 @@ jobs:
           yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd
           echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc
 
-      # Need this for virtualenv and arrow tests if enabled
-      - name: 'Install Python'
-        uses: actions/setup-python@v4
-        if: ${{ !inputs.manylinux }}
-        with:
-          python-version: '3.9'
-          cache: 'pip'
-
       # This must happen after checkout, because checkout would remove the directory.
       - name: Install Ninja
         if: inputs.cmake_generator == 'Ninja'
         uses: seanmiddleditch/gha-setup-ninja@v4
 
-      - name: 'Set up Python dependencies'
-        if: ${{ !inputs.manylinux }}
-        run: |
-          set -e pipefail
-          python -m pip install --upgrade pip virtualenv
-          pip install pyarrow pybind11 numpy
-
       - name: 'Brew setup on macOS' # x-ref c8e49ba8f8b9ce
         if: ${{ startsWith(matrix.os, 'macos-') == true }}
         run: |
@@ -217,7 +200,7 @@ jobs:
           ./test/ci/test_assert -d yes
           ./test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
           ./tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
-          
+
           ctest -R tiledb_timing_unit | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1'
 
           ###################################################

.github/workflows/release.yml

@@ -208,5 +208,5 @@ jobs:
         uses: TileDB-Inc/github-actions/open-issue@main
         with:
           name: Release failed
-          label: bug
+          label: release
           assignee: KiterLuc,teo-tsirpanis,davisp

.github/workflows/unit-test-runs.yml

@@ -27,6 +27,19 @@ jobs:
       - name: Install Ninja
         uses: seanmiddleditch/gha-setup-ninja@v4
 
+      # Need this for virtualenv and arrow tests if enabled
+      - name: 'Install Python'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+          cache: 'pip'
+
+      - name: 'Set up Python dependencies'
+        run: |
+          set -e pipefail
+          python -m pip install --upgrade pip virtualenv
+          pip install pyarrow pybind11[global] numpy
+
       - name: Setup MSVC toolset
         uses: TheMrMilchmann/setup-msvc-dev@v3
         if: ${{ startsWith(matrix.os, 'windows-') }}
@@ -60,7 +73,8 @@ jobs:
             -G Ninja \
             -DCMAKE_BUILD_TYPE=Debug \
             -DTILEDB_SERIALIZATION=ON \
-            -DTILEDB_ASSERTIONS=ON
+            -DTILEDB_ASSERTIONS=ON \
+            -DTILEDB_ARROW_TESTS=ON
           # Build all unit tests
           cmake --build build --target tests -j4
 

CMakeLists.txt

@@ -286,6 +286,12 @@ endif()
 
 list(APPEND TILEDB_C_API_RELATIVE_HEADERS
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/api_external_common.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array/array_api_external.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_schema_api_deprecated.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_schema_api_experimental.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_schema_api_external.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_type_enum.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/layout_enum.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/attribute/attribute_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/attribute/attribute_api_external_experimental.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/buffer/buffer_api_external.h"
@@ -515,12 +521,16 @@ if (TILEDB_TESTS)
   add_dependencies(tests tiledb_unit)
   add_dependencies(tests tiledb_regression)
   add_dependencies(tests test_assert)
+  if(TILEDB_ARROW_TESTS)
+    add_dependencies(tests unit_arrow)
+  endif()
 
   # C API support
   add_dependencies(tests unit_capi_handle unit_capi_exception_wrapper)
   # C API basics
   add_dependencies(tests unit_capi_config unit_capi_context)
   add_dependencies(tests unit_capi_array)
+  add_dependencies(tests unit_capi_array_schema)
   add_dependencies(tests unit_capi_buffer)
   add_dependencies(tests unit_capi_buffer_list)
   add_dependencies(tests unit_capi_data_order)

HISTORY.md

@@ -1,3 +1,67 @@
+# TileDB v2.25.0 Release Notes
+
+## Announcements
+
+* TileDB 2.25, includes the new current domain feature which allows to specify an area of the domain that is considered to be active for sparse arrays.
+
+## Deprecation announcements
+
+* The HDFS backend is no longer officially tested by TileDB. As announced before, it is scheduled to be removed in version 2.28, to be released in Q4 2024. [#5085](https://github.com/TileDB-Inc/TileDB/pull/5085)
+* Support for reading sparse fragments in dense arrays will be removed in version 2.27. Writting sparse fragments in dense arrays was removed in version 2.5. [#5116](https://github.com/TileDB-Inc/TileDB/pull/5116)
+* Support for returning the same results multiple times in sparse reads when ranges overlap will be removed in version 2.27. This was possible by setting `sm.merge_overlapping_ranges_experimental` to `false`, but the default `true` behavior has been there since version 2.17.
+
+## New features
+
+* REST support for current domain. [#5136](https://github.com/TileDB-Inc/TileDB/pull/5136)
+* Disallow writing outside of the current domain. [#5165](https://github.com/TileDB-Inc/TileDB/pull/5165)
+* Current domain: disallow reading outside of current domain. [#5168](https://github.com/TileDB-Inc/TileDB/pull/5168)
+
+## Improvements
+
+* Improve memory consumption for tile structures in dense reader. [#5046](https://github.com/TileDB-Inc/TileDB/pull/5046)
+
+## Defects removed
+
+* Fail early when trying to add members with relative URIs in remote groups. [#5025](https://github.com/TileDB-Inc/TileDB/pull/5025)
+* Correct defective return value in `Posix::ls_with_sizes`. [#5037](https://github.com/TileDB-Inc/TileDB/pull/5037)
+* Prevent constructing attribute with invalid cell_val_num. [#4952](https://github.com/TileDB-Inc/TileDB/pull/4952)
+* Do not mask failures when listing a directory fails on POSIX. [#5043](https://github.com/TileDB-Inc/TileDB/pull/5043)
+* Fix write queries using `sm.var_offsets.extra_element=true`. [#5033](https://github.com/TileDB-Inc/TileDB/pull/5033)
+* Fix segfaults in WebP queries ran in parallel. [#5065](https://github.com/TileDB-Inc/TileDB/pull/5065)
+* Fix exceptions with message: ```unknown exception type; no further information```. [#5080](https://github.com/TileDB-Inc/TileDB/pull/5080)
+* Fix check for out of bounds dimension in Dimension::dimension_ptr. [#5094](https://github.com/TileDB-Inc/TileDB/pull/5094)
+* Fix array latest schema selection for same MS timestamps schemas. [#5143](https://github.com/TileDB-Inc/TileDB/pull/5143)
+* Fix serialization issue with schema evolution for query v3. [#5154](https://github.com/TileDB-Inc/TileDB/pull/5154)
+
+## Configuration changes
+
+* Add `vfs.s3.storage_class` config option to set the storage class of newly uploaded S3 objects. [#5053](https://github.com/TileDB-Inc/TileDB/pull/5053)
+* Add `rest.custom_headers.*` config option to set custom headers on REST requests. [#5104](https://github.com/TileDB-Inc/TileDB/pull/5104)
+* Add `rest.payer_namespace` config option to set the namespace to be charged for REST requests. [#5105](https://github.com/TileDB-Inc/TileDB/pull/5105)
+
+## API changes
+
+### C API
+
+* Add CurrentDomain API support. [#5041](https://github.com/TileDB-Inc/TileDB/pull/5041)
+
+### C++ API
+
+* Current Domain CPP API implementation. [#5056](https://github.com/TileDB-Inc/TileDB/pull/5056)
+
+## Build System Changes
+
+* Backwards compatibility with older CMake versions for libfaketime. [#5049](https://github.com/TileDB-Inc/TileDB/pull/5049)
+* Automatic downloading of vcpkg can be disabled by enabling the `TILEDB_DISABLE_AUTO_VCPKG` CMake option, in addition to setting the environment variable with trhe same name. [#5048](https://github.com/TileDB-Inc/TileDB/pull/5048)
+* Improve embedding of `magic.mgc` and allow compiling with any libmagic version. [#4989](https://github.com/TileDB-Inc/TileDB/pull/4989)
+
+## Internal Improvements
+
+* Implement actualize function that orders data underlying `alt_var_length_view` [#5087](https://github.com/TileDB-Inc/TileDB/pull/5087)
+* Implement a partitioning function to partition cells to fit into fixed size bins [#5092](https://github.com/TileDB-Inc/TileDB/pull/5092)
+* Implementation of a `chunk_view` class to provide a subset of C++23 chunk_view, suitable for supporting external sort. [#5035](https://github.com/TileDB-Inc/TileDB/pull/5035)
+* Tests that the chunks in a `chunk_view` can be separately sorted. [#5052](https://github.com/TileDB-Inc/TileDB/pull/5052)
+
 # TileDB v2.24.2 Release Notes
 
 ## Defects removed

cmake/Options/TileDBToolchain.cmake

@@ -36,7 +36,7 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE)
         # Inspired from https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.10.3/cmake-modules/AzureVcpkg.cmake
         message("TILEDB_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.")
         # To help with resolving conflicts, when you update the commit, also update its date.
-        set(VCPKG_COMMIT_STRING 72010900b7cee36cea77aebb97695095c9358eaf) # 2023-12-05
+        set(VCPKG_COMMIT_STRING 7aeffc91033ad35cc4e2c152f213a866ec6c11ac) # 2024-07-13
         message("Vcpkg commit string used: ${VCPKG_COMMIT_STRING}")
         include(FetchContent)
         FetchContent_Declare(

examples/c_api/current_domain.c

@@ -74,7 +74,7 @@ void create_array() {
   tiledb_ndrectangle_set_range_for_name(ctx, ndrect, "d1", &range);
 
   // Assign the rectangle to the current domain
-  tiledb_current_domain_set_ndrectangle(current_domain, ndrect);
+  tiledb_current_domain_set_ndrectangle(ctx, current_domain, ndrect);
 
   // Create a single attribute "a" so each cell can store an integer
   tiledb_attribute_t* a;
@@ -119,21 +119,21 @@ void print_current_domain() {
 
   // Check if current domain is empty
   uint32_t is_empty;
-  tiledb_current_domain_get_is_empty(current_domain, &is_empty);
+  tiledb_current_domain_get_is_empty(ctx, current_domain, &is_empty);
 
   if (is_empty) {
     printf("Current domain: empty\n");
   } else {
     // Get current domain type
     tiledb_current_domain_type_t current_domain_type;
-    tiledb_current_domain_get_type(current_domain, &current_domain_type);
+    tiledb_current_domain_get_type(ctx, current_domain, &current_domain_type);
 
     if (current_domain_type == TILEDB_NDRECTANGLE) {
       printf("Current domain type: NDRECTANGLE\n");
 
       // Get the ND rectangle
       tiledb_ndrectangle_t* ndrect;
-      tiledb_current_domain_get_ndrectangle(current_domain, &ndrect);
+      tiledb_current_domain_get_ndrectangle(ctx, current_domain, &ndrect);
 
       tiledb_range_t range;
       tiledb_ndrectangle_get_range_from_name(ctx, ndrect, "d1", &range);
@@ -143,6 +143,23 @@ void print_current_domain() {
           *(int*)range.min,
           *(int*)range.max);
 
+      // Get datatype of range
+      tiledb_datatype_t dtype;
+      tiledb_ndrectangle_get_dtype(ctx, ndrect, 0, &dtype);
+      const char* dtype_str;
+      tiledb_datatype_to_str(dtype, &dtype_str);
+      printf("Range 0 dtype: %s\n", dtype_str);
+
+      // Get datatype of range by name
+      tiledb_ndrectangle_get_dtype_from_name(ctx, ndrect, "d1", &dtype);
+      tiledb_datatype_to_str(dtype, &dtype_str);
+      printf("Range 0 dtype by name: %s\n", dtype_str);
+
+      // Get dim num
+      uint32_t ndim;
+      tiledb_ndrectangle_get_dim_num(ctx, ndrect, &ndim);
+      printf("Range 0 dtype by name: %d\n", ndim);
+
       // Clean up
       tiledb_ndrectangle_free(&ndrect);
     } else {
@@ -191,7 +208,7 @@ void expand_current_domain() {
   tiledb_ndrectangle_set_range_for_name(ctx, ndrect, "d1", &range);
 
   // Set the rectangle to the current domain
-  tiledb_current_domain_set_ndrectangle(new_current_domain, ndrect);
+  tiledb_current_domain_set_ndrectangle(ctx, new_current_domain, ndrect);
 
   // Expand the current domain
   tiledb_array_schema_evolution_expand_current_domain(

examples/cpp_api/current_domain.cc

@@ -113,6 +113,17 @@ void print_current_domain(Context& ctx) {
   // Print the range
   std::cout << "Current domain range: [" << range[0] << ", " << range[1] << "]"
             << std::endl;
+
+  // Print datatype of range 0
+  std::cout << "Current domain range 0 datatype: "
+            << tiledb::impl::type_to_str(ndrect.range_dtype(0)) << std::endl;
+
+  // Print datatype of range d1
+  std::cout << "Current domain range 0 datatype: "
+            << tiledb::impl::type_to_str(ndrect.range_dtype("d1")) << std::endl;
+
+  // Print dim num
+  std::cout << "Current domain dim num: " << ndrect.dim_num() << std::endl;
 }
 
 void expand_current_domain(Context& ctx) {