From 12e474f809c27a930425488c2e260179b8aeb4b7 Mon Sep 17 00:00:00 2001 From: Agisilaos Kounelis Date: Mon, 2 Sep 2024 15:19:44 +0300 Subject: [PATCH] Merge branch 'dev' into de/sc-27472/arrow_nullable --- ...ld-ubuntu20.04-backwards-compatibility.yml | 2 +- .github/workflows/build-windows.yml | 37 +- .github/workflows/ci-linux_mac.yml | 19 +- .github/workflows/release.yml | 2 +- .github/workflows/unit-test-runs.yml | 16 +- CMakeLists.txt | 10 + HISTORY.md | 64 + cmake/Options/TileDBToolchain.cmake | 2 +- examples/c_api/current_domain.c | 27 +- examples/cpp_api/current_domain.cc | 11 + .../data_block/test/unit_blocks_and_ports.cc | 1 + .../dag/data_block/test/unit_data_block.cc | 1 + .../tiledb/common/dag/edge/test/unit_edge.cc | 1 + .../common/dag/execution/test/unit_duffs.cc | 4 +- .../common/dag/execution/test/unit_frugal.cc | 1 + .../dag/execution/test/unit_scheduler.cc | 1 + .../common/dag/graph/test/unit_taskgraph.cc | 4 +- .../tiledb/common/dag/nodes/generators.h | 1 + .../dag/nodes/test/unit_general_nodes.cc | 1 + .../nodes/test/unit_segmented_mimo_nodes.cc | 2 + .../dag/nodes/test/unit_simple_nodes.cc | 1 + .../common/dag/nodes/test/unit_stop_source.cc | 1 + .../dag/nodes/test/unit_util_functions.cc | 1 + .../common/dag/ports/test/unit_ports.cc | 1 + .../dag/utility/test/unit_bounded_buffer.cc | 1 + format_spec/FORMAT_SPEC.md | 30 +- format_spec/history.md | 169 ++ format_spec/vacuum_file.md | 2 +- ports/README.md | 16 +- ports/aws-c-io/fix-pointer-mismatch.patch | 22 + ports/aws-c-io/portfile.cmake | 34 + ports/aws-c-io/vcpkg.json | 24 + ports/aws-sdk-cpp/find-dependency.patch | 14 + ports/aws-sdk-cpp/fix-aws-root.patch | 22 + ports/aws-sdk-cpp/fix-winsock-headers.patch | 14 + ports/aws-sdk-cpp/fix_find_curl.patch | 31 + ports/aws-sdk-cpp/generateFeatures.ps1 | 73 + .../lock-curl-http-and-tls-settings.patch | 20 + ports/aws-sdk-cpp/no-test-deps.patch | 25 + ports/aws-sdk-cpp/portfile.cmake | 104 ++ ports/aws-sdk-cpp/usage | 12 + ports/aws-sdk-cpp/vcpkg.in.json | 37 + ports/aws-sdk-cpp/vcpkg.json | 1276 ++++++++++++++ ports/azure-storage-common-cpp/portfile.cmake | 32 - ports/azure-storage-common-cpp/vcpkg.json | 36 - ports/google-cloud-cpp/portfile.cmake | 86 - ports/google-cloud-cpp/remove_gmock.patch | 28 - .../google-cloud-cpp/support_absl_cxx17.patch | 19 - ports/google-cloud-cpp/vcpkg.json | 1509 ----------------- ports/pcre2/fix-cmake.patch | 334 ---- ports/pcre2/no-static-suffix.patch | 33 - ports/pcre2/pcre2-10.35_fix-uwp.patch | 10 - ports/pcre2/portfile.cmake | 73 - ports/pcre2/usage | 6 - ports/pcre2/vcpkg.json | 39 - ports/triplets/arm64-osx-asan.cmake | 6 + ports/triplets/arm64-osx-release.cmake | 6 + ports/triplets/arm64-osx-relwithdebinfo.cmake | 6 + ports/triplets/arm64-osx.cmake | 6 + ports/triplets/x64-linux-asan.cmake | 6 + ports/triplets/x64-linux-release.cmake | 6 + ports/triplets/x64-linux-relwithdebinfo.cmake | 6 + ports/triplets/x64-linux.cmake | 6 + ports/triplets/x64-osx-asan.cmake | 6 + ports/triplets/x64-osx-release.cmake | 6 + ports/triplets/x64-osx-relwithdebinfo.cmake | 6 + ports/triplets/x64-osx.cmake | 6 + ports/vcpkg-cmake-config/copyright | 23 - ports/vcpkg-cmake-config/portfile.cmake | 12 - .../vcpkg-port-config.cmake | 1 - ports/vcpkg-cmake-config/vcpkg.json | 6 - .../vcpkg_cmake_config_fixup.cmake | 266 --- scripts/install-gcs-emu.sh | 2 +- scripts/install-run-gcs-emu.ps1 | 2 +- test/CMakeLists.txt | 128 +- ...it-cppapi-consolidation-with-timestamps.cc | 45 - test/regression/CMakeLists.txt | 4 + test/regression/targets/sc-52975.cc | 354 ++++ test/regression/targets/sc-53334.cc | 194 +++ test/regression/targets/sc-53791.cc | 141 ++ test/regression/targets/sc-53970.cc | 127 ++ ...est-capi-array-write-ordered-attr-fixed.cc | 6 +- .../test-capi-array-write-ordered-attr-var.cc | 6 +- test/src/test-capi-dimension-label.cc | 12 +- test/src/test-capi-subarray-labels.cc | 5 +- test/src/test-cppapi-current-domain.cc | 7 + test/src/unit-arrow.cc | 96 +- test/src/unit-capi-array.cc | 2 + test/src/unit-capi-array_schema.cc | 13 +- test/src/unit-capi-config.cc | 2 +- test/src/unit-capi-enum_values.cc | 3 +- test/src/unit-capi-fragment_info.cc | 35 +- ...it-cppapi-consolidation-with-timestamps.cc | 54 +- test/src/unit-cppapi-deletes.cc | 58 +- test/src/unit-cppapi-fragment_info.cc | 14 +- ...nit-cppapi-query-condition-enumerations.cc | 1 + test/src/unit-cppapi-schema-evolution.cc | 2 +- test/src/unit-curl.cc | 33 +- test/src/unit-current-domain-rest.cc | 10 +- test/src/unit-enumerations.cc | 64 + test/src/unit-request-handlers.cc | 114 +- test/src/unit-sparse-global-order-reader.cc | 1 + .../unit-sparse-unordered-with-dups-reader.cc | 1 + test/src/unit_arrow.py | 78 +- test/support/src/helpers.cc | 20 + test/support/src/helpers.h | 9 + test/support/src/serialization_wrappers.cc | 5 +- tiledb/api/c_api/CMakeLists.txt | 5 +- tiledb/api/c_api/array/CMakeLists.txt | 13 + tiledb/api/c_api/array/array_api.cc | 101 ++ tiledb/api/c_api/array/array_api_external.h | 104 ++ tiledb/api/c_api/array/array_api_internal.h | 40 + tiledb/api/c_api/array/test/CMakeLists.txt | 2 +- .../test/compile_capi_array_stub_main.cc} | 9 +- .../api/c_api/array/test/unit_capi_array.cc | 29 +- tiledb/api/c_api/array_schema/CMakeLists.txt | 42 + .../c_api/array_schema/array_schema_api.cc | 740 ++++++++ .../array_schema_api_deprecated.h | 68 + .../array_schema_api_experimental.h | 156 ++ .../array_schema/array_schema_api_external.h | 684 ++++++++ .../array_schema/array_schema_api_internal.h | 259 +++ .../api/c_api/array_schema/array_type_enum.h | 41 + tiledb/api/c_api/array_schema/layout_enum.h | 47 + .../c_api/array_schema/test/CMakeLists.txt | 37 + .../compile_capi_array_schema_stub_main.cc | 35 + .../test/unit_capi_array_schema.cc | 855 ++++++++++ tiledb/api/c_api/config/config_api_external.h | 6 +- tiledb/api/c_api/context/CMakeLists.txt | 3 +- .../current_domain/current_domain_api.cc | 21 +- ...current_domain_api_external_experimental.h | 16 +- .../test/compile_capi_current_domain_main.cc | 8 +- .../test/unit_capi_current_domain.cc | 46 +- .../api/c_api/ndrectangle/ndrectangle_api.cc | 106 +- .../ndrectangle_api_external_experimental.h | 68 + .../ndrectangle/test/unit_capi_ndrectangle.cc | 39 + .../testsupport_capi_array_schema.h | 93 + tiledb/common/common-std.h | 7 +- tiledb/doxygen/source/conf.py | 4 +- tiledb/sm/array/array.cc | 22 +- tiledb/sm/array/array.h | 4 +- tiledb/sm/array/array_directory.cc | 2 +- tiledb/sm/array_schema/CMakeLists.txt | 2 +- tiledb/sm/array_schema/array_schema.cc | 2 +- .../array_schema/array_schema_operations.cc | 61 + .../sm/array_schema/array_schema_operations.h | 13 + tiledb/sm/array_schema/ndrectangle.cc | 14 + tiledb/sm/array_schema/ndrectangle.h | 18 + tiledb/sm/c_api/api_argument_validator.h | 13 +- tiledb/sm/c_api/tiledb.cc | 1088 +----------- tiledb/sm/c_api/tiledb.h | 660 +------ tiledb/sm/c_api/tiledb_deprecated.h | 23 +- tiledb/sm/c_api/tiledb_dimension_label.cc | 57 +- tiledb/sm/c_api/tiledb_enum.h | 24 +- tiledb/sm/c_api/tiledb_experimental.h | 108 +- tiledb/sm/c_api/tiledb_filestore.cc | 20 +- tiledb/sm/c_api/tiledb_struct_def.h | 6 +- tiledb/sm/c_api/tiledb_version.h | 2 +- tiledb/sm/config/config.cc | 2 +- tiledb/sm/cpp_api/config.h | 4 + tiledb/sm/cpp_api/current_domain.h | 14 +- tiledb/sm/cpp_api/ndrectangle.h | 46 + tiledb/sm/crypto/CMakeLists.txt | 7 +- tiledb/sm/crypto/crypto.cc | 20 +- tiledb/sm/crypto/crypto_openssl.cc | 29 +- tiledb/sm/crypto/crypto_openssl.h | 10 - tiledb/sm/crypto/crypto_win32.cc | 41 +- tiledb/sm/crypto/crypto_win32.h | 22 +- tiledb/sm/crypto/test/CMakeLists.txt | 32 + .../sm/crypto/test/unit_tiledb_crypto.cc | 210 ++- tiledb/sm/enums/array_type.h | 10 +- tiledb/sm/enums/layout.h | 10 +- tiledb/sm/filesystem/s3.cc | 57 +- tiledb/sm/filesystem/s3.h | 18 +- tiledb/sm/filter/filter_pipeline.h | 5 +- .../test/unit_bit_width_reduction_pipeline.cc | 1 + tiledb/sm/filter/test/unit_xor_pipeline.cc | 1 + tiledb/sm/fragment/fragment_metadata.cc | 19 +- tiledb/sm/fragment/fragment_metadata.h | 3 - .../sm/fragment/loaded_fragment_metadata.cc | 7 +- tiledb/sm/fragment/loaded_fragment_metadata.h | 2 +- tiledb/sm/fragment/single_fragment_info.h | 1 + tiledb/sm/misc/rectangle.cc | 4 - tiledb/sm/misc/rectangle.h | 1 + tiledb/sm/query/query_condition.h | 3 +- tiledb/sm/query/readers/dense_reader.cc | 2 +- tiledb/sm/query/readers/dense_reader.h | 8 +- tiledb/sm/rest/curl.cc | 14 +- tiledb/sm/rest/curl.h | 6 +- tiledb/sm/rest/rest_client.h | 5 +- tiledb/sm/rest/rest_client_remote.cc | 17 +- tiledb/sm/rest/rest_client_remote.h | 13 +- tiledb/sm/serialization/array_schema.cc | 83 +- tiledb/sm/serialization/array_schema.h | 16 +- tiledb/sm/serialization/tiledb-rest.capnp | 5 + tiledb/sm/serialization/tiledb-rest.capnp.c++ | 60 +- tiledb/sm/serialization/tiledb-rest.capnp.h | 110 +- .../serialization/serializers.h | 15 + vcpkg.json | 13 +- 198 files changed, 7910 insertions(+), 5229 deletions(-) create mode 100644 format_spec/history.md create mode 100644 ports/aws-c-io/fix-pointer-mismatch.patch create mode 100644 ports/aws-c-io/portfile.cmake create mode 100644 ports/aws-c-io/vcpkg.json create mode 100644 ports/aws-sdk-cpp/find-dependency.patch create mode 100644 ports/aws-sdk-cpp/fix-aws-root.patch create mode 100644 ports/aws-sdk-cpp/fix-winsock-headers.patch create mode 100644 ports/aws-sdk-cpp/fix_find_curl.patch create mode 100644 ports/aws-sdk-cpp/generateFeatures.ps1 create mode 100644 ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch create mode 100644 ports/aws-sdk-cpp/no-test-deps.patch create mode 100644 ports/aws-sdk-cpp/portfile.cmake create mode 100644 ports/aws-sdk-cpp/usage create mode 100644 ports/aws-sdk-cpp/vcpkg.in.json create mode 100644 ports/aws-sdk-cpp/vcpkg.json delete mode 100644 ports/azure-storage-common-cpp/portfile.cmake delete mode 100644 ports/azure-storage-common-cpp/vcpkg.json delete mode 100644 ports/google-cloud-cpp/portfile.cmake delete mode 100644 ports/google-cloud-cpp/remove_gmock.patch delete mode 100644 ports/google-cloud-cpp/support_absl_cxx17.patch delete mode 100644 ports/google-cloud-cpp/vcpkg.json delete mode 100644 ports/pcre2/fix-cmake.patch delete mode 100644 ports/pcre2/no-static-suffix.patch delete mode 100644 ports/pcre2/pcre2-10.35_fix-uwp.patch delete mode 100644 ports/pcre2/portfile.cmake delete mode 100644 ports/pcre2/usage delete mode 100644 ports/pcre2/vcpkg.json delete mode 100644 ports/vcpkg-cmake-config/copyright delete mode 100644 ports/vcpkg-cmake-config/portfile.cmake delete mode 100644 ports/vcpkg-cmake-config/vcpkg-port-config.cmake delete mode 100644 ports/vcpkg-cmake-config/vcpkg.json delete mode 100644 ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake create mode 100644 test/regression/targets/sc-52975.cc create mode 100644 test/regression/targets/sc-53334.cc create mode 100644 test/regression/targets/sc-53791.cc create mode 100644 test/regression/targets/sc-53970.cc create mode 100644 tiledb/api/c_api/array/array_api.cc create mode 100644 tiledb/api/c_api/array/array_api_external.h create mode 100644 tiledb/api/c_api/array/array_api_internal.h rename tiledb/{sm/crypto/test/compile_crypto_main.cc => api/c_api/array/test/compile_capi_array_stub_main.cc} (84%) create mode 100644 tiledb/api/c_api/array_schema/CMakeLists.txt create mode 100644 tiledb/api/c_api/array_schema/array_schema_api.cc create mode 100644 tiledb/api/c_api/array_schema/array_schema_api_deprecated.h create mode 100644 tiledb/api/c_api/array_schema/array_schema_api_experimental.h create mode 100644 tiledb/api/c_api/array_schema/array_schema_api_external.h create mode 100644 tiledb/api/c_api/array_schema/array_schema_api_internal.h create mode 100644 tiledb/api/c_api/array_schema/array_type_enum.h create mode 100644 tiledb/api/c_api/array_schema/layout_enum.h create mode 100644 tiledb/api/c_api/array_schema/test/CMakeLists.txt create mode 100644 tiledb/api/c_api/array_schema/test/compile_capi_array_schema_stub_main.cc create mode 100644 tiledb/api/c_api/array_schema/test/unit_capi_array_schema.cc create mode 100644 tiledb/api/c_api_test_support/testsupport_capi_array_schema.h create mode 100644 tiledb/sm/crypto/test/CMakeLists.txt rename test/src/unit-crypto.cc => tiledb/sm/crypto/test/unit_tiledb_crypto.cc (76%) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index 6a9355830854..20d7cc574148 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0", "v2_24_0"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0", "v2_24_0", "v2_25_0"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 7b26ecee8c79..70b3a43fb7fb 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -24,19 +24,16 @@ jobs: include: - environ: 'azure' TILEDB_AZURE: ON - TILEDB_ARROW_TESTS: ON TILEDB_SERIALIZATION: OFF TILEDB_S3: OFF TILEDB_WEBP: OFF - environ: 's3' TILEDB_S3: ON - TILEDB_ARROW_TESTS: ON TILEDB_SERIALIZATION: OFF TILEDB_AZURE: OFF TILEDB_WEBP: OFF - environ: 'gcs' TILEDB_GCS: ON - TILEDB_ARROW_TESTS: ON TILEDB_SERIALIZATION: OFF TILEDB_AZURE: OFF TILEDB_S3: OFF @@ -44,7 +41,6 @@ jobs: - environ: 'serialization' TILEDB_S3: OFF TILEDB_AZURE: OFF - TILEDB_ARROW_TESTS: OFF TILEDB_SERIALIZATION: ON TILEDB_WEBP: ON @@ -57,7 +53,6 @@ jobs: TILEDB_AZURE: ${{ matrix.TILEDB_AZURE }} #azure }} TILEDB_GCS: ${{ matrix.TILEDB_GCS }} #gcs }} TILEDB_SERIALIZATION: ${{ matrix.TILEDB_SERIALIZATION }} #serialization }} - TILEDB_ARROW_TESTS: ${{ matrix.TILEDB_ARROW_TESTS }} TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }} TILEDB_CMAKE_BUILD_TYPE: 'Release' VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' @@ -110,15 +105,6 @@ jobs: #but, it seems *we* can populate our own var from our actions, and access it in file upload... echo TDBLOCALAPPDATA=$env:LOCALAPPDATA >> "$env:GITHUB_ENV" - - name: ARROW python needs - shell: bash - if: ${{ matrix.TILEDB_ARROW_TESTS == 'ON' }} - run: | - set -e pipefail - if [[ "$TILEDB_ARROW_TESTS" == "ON" ]]; then - pip install pyarrow pybind11 numpy - fi - - name: Prepare git run: git config --global core.autocrlf false @@ -167,6 +153,9 @@ jobs: if ($env:TILEDB_TOOLS -eq "ON") { $bootstrapOptions = $bootstrapOptions + " -EnableTools" } + if ($env:TILEDB_ARROW_TESTS -eq "ON") { + $bootstrapOptions = $bootstrapOptions + " -EnableArrowTests" + } $CMakeBuildType = $env:TILEDB_CMAKE_BUILD_TYPE if ($env:TILEDB_DEBUG -eq "ON") { $bootstrapOptions = $bootstrapOptions + " -EnableDebug" @@ -183,10 +172,6 @@ jobs: # if ($env:TILEDB_CI_TSAN -eq "ON") { # $bootstrapOptions = $bootstrapOptions + " -EnableSanitizer thread -EnableDebug" # } - # static already added above as initial default - # if ($env:TILEDB_FORCE_BUILD_DEPS" -eq "ON") { - # $bootstrapOptions = $bootstrapOptions + " -EnableBuildDeps" - # } if ($env:TILEDB_WERROR -eq "OFF") { $bootstrapOptions = $bootstrapOptions + " -DisableWerror" } @@ -256,7 +241,7 @@ jobs: } else { #using the node/npm already present in vm Write-Host "azurite: using vm install nodejs" - #this code path avoids overhead of download/expand/install of alternate nodejs/azurite. + #this code path avoids overhead of download/expand/install of alternate nodejs/azurite. npm install -g azurite Write-Host "done with 'npm install -g azurite'" $azuriteDataPath = (Join-Path $env:TEMP "azuriteData") @@ -281,23 +266,23 @@ jobs: Write-Host "cmds: '$cmds'" Invoke-Expression $cmds if ($LastExitCode -ne 0) { - Write-Host "Tests failed. tiledb_unit exit status: " $LastExitCocde - $host.SetShouldExit($LastExitCode) + Write-Host "Tests failed. tiledb_unit exit status: " $LastExitCocde + $host.SetShouldExit($LastExitCode) } - + $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\sm\filesystem\test\unit_vfs -d=yes" Write-Host "cmds: '$cmds'" Invoke-Expression $cmds if ($LastExitCode -ne 0) { - Write-Host "Tests failed. tiledb_vfs exit status: " $LastExitCocde - $host.SetShouldExit($LastExitCode) + Write-Host "Tests failed. tiledb_vfs exit status: " $LastExitCocde + $host.SetShouldExit($LastExitCode) } $cmds = "$env:BUILD_BUILDDIRECTORY\test\ci\test_assert.exe -d=yes" Invoke-Expression $cmds if ($LastExitCode -ne 0) { - Write-Host "Tests failed. test_assert exit status: " $LastExitCocde - $host.SetShouldExit($LastExitCode) + Write-Host "Tests failed. test_assert exit status: " $LastExitCocde + $host.SetShouldExit($LastExitCode) } - name: Build examples shell: pwsh diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index 869275e89b68..2428972ee31a 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -72,8 +72,6 @@ env: CMAKE_GENERATOR: ${{ inputs.cmake_generator }} TILEDB_CI_BACKEND: ${{ inputs.ci_backend }} TILEDB_CI_OS: ${{ startsWith(inputs.matrix_image, 'ubuntu-') && 'Linux' || 'macOS' }} - # Installing Python does not work on manylinux. - TILEDB_ARROW_TESTS: ${{ !inputs.manylinux && 'ON' || 'OFF' }} TILEDB_MANYLINUX: ${{ !inputs.manylinux && 'ON' || 'OFF' }} CXX: ${{ inputs.matrix_compiler_cxx }} CC: ${{ inputs.matrix_compiler_cc }} @@ -130,26 +128,11 @@ jobs: yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc - # Need this for virtualenv and arrow tests if enabled - - name: 'Install Python' - uses: actions/setup-python@v4 - if: ${{ !inputs.manylinux }} - with: - python-version: '3.9' - cache: 'pip' - # This must happen after checkout, because checkout would remove the directory. - name: Install Ninja if: inputs.cmake_generator == 'Ninja' uses: seanmiddleditch/gha-setup-ninja@v4 - - name: 'Set up Python dependencies' - if: ${{ !inputs.manylinux }} - run: | - set -e pipefail - python -m pip install --upgrade pip virtualenv - pip install pyarrow pybind11 numpy - - name: 'Brew setup on macOS' # x-ref c8e49ba8f8b9ce if: ${{ startsWith(matrix.os, 'macos-') == true }} run: | @@ -217,7 +200,7 @@ jobs: ./test/ci/test_assert -d yes ./test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' ./tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' - + ctest -R tiledb_timing_unit | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' ################################################### diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 935497ffc126..eaee179e12b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -208,5 +208,5 @@ jobs: uses: TileDB-Inc/github-actions/open-issue@main with: name: Release failed - label: bug + label: release assignee: KiterLuc,teo-tsirpanis,davisp diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml index b0f3f25bc083..a56c4aca174f 100644 --- a/.github/workflows/unit-test-runs.yml +++ b/.github/workflows/unit-test-runs.yml @@ -27,6 +27,19 @@ jobs: - name: Install Ninja uses: seanmiddleditch/gha-setup-ninja@v4 + # Need this for virtualenv and arrow tests if enabled + - name: 'Install Python' + uses: actions/setup-python@v5 + with: + python-version: '3.9' + cache: 'pip' + + - name: 'Set up Python dependencies' + run: | + set -e pipefail + python -m pip install --upgrade pip virtualenv + pip install pyarrow pybind11[global] numpy + - name: Setup MSVC toolset uses: TheMrMilchmann/setup-msvc-dev@v3 if: ${{ startsWith(matrix.os, 'windows-') }} @@ -60,7 +73,8 @@ jobs: -G Ninja \ -DCMAKE_BUILD_TYPE=Debug \ -DTILEDB_SERIALIZATION=ON \ - -DTILEDB_ASSERTIONS=ON + -DTILEDB_ASSERTIONS=ON \ + -DTILEDB_ARROW_TESTS=ON # Build all unit tests cmake --build build --target tests -j4 diff --git a/CMakeLists.txt b/CMakeLists.txt index d3fec8f854cd..f43f33f3b2e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -286,6 +286,12 @@ endif() list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/api_external_common.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array/array_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_schema_api_deprecated.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_schema_api_experimental.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_schema_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_type_enum.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/layout_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/attribute/attribute_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/attribute/attribute_api_external_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/buffer/buffer_api_external.h" @@ -515,12 +521,16 @@ if (TILEDB_TESTS) add_dependencies(tests tiledb_unit) add_dependencies(tests tiledb_regression) add_dependencies(tests test_assert) + if(TILEDB_ARROW_TESTS) + add_dependencies(tests unit_arrow) + endif() # C API support add_dependencies(tests unit_capi_handle unit_capi_exception_wrapper) # C API basics add_dependencies(tests unit_capi_config unit_capi_context) add_dependencies(tests unit_capi_array) + add_dependencies(tests unit_capi_array_schema) add_dependencies(tests unit_capi_buffer) add_dependencies(tests unit_capi_buffer_list) add_dependencies(tests unit_capi_data_order) diff --git a/HISTORY.md b/HISTORY.md index 4b78cef9702f..096b936e4d48 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,67 @@ +# TileDB v2.25.0 Release Notes + +## Announcements + +* TileDB 2.25, includes the new current domain feature which allows to specify an area of the domain that is considered to be active for sparse arrays. + +## Deprecation announcements + +* The HDFS backend is no longer officially tested by TileDB. As announced before, it is scheduled to be removed in version 2.28, to be released in Q4 2024. [#5085](https://github.com/TileDB-Inc/TileDB/pull/5085) +* Support for reading sparse fragments in dense arrays will be removed in version 2.27. Writting sparse fragments in dense arrays was removed in version 2.5. [#5116](https://github.com/TileDB-Inc/TileDB/pull/5116) +* Support for returning the same results multiple times in sparse reads when ranges overlap will be removed in version 2.27. This was possible by setting `sm.merge_overlapping_ranges_experimental` to `false`, but the default `true` behavior has been there since version 2.17. + +## New features + +* REST support for current domain. [#5136](https://github.com/TileDB-Inc/TileDB/pull/5136) +* Disallow writing outside of the current domain. [#5165](https://github.com/TileDB-Inc/TileDB/pull/5165) +* Current domain: disallow reading outside of current domain. [#5168](https://github.com/TileDB-Inc/TileDB/pull/5168) + +## Improvements + +* Improve memory consumption for tile structures in dense reader. [#5046](https://github.com/TileDB-Inc/TileDB/pull/5046) + +## Defects removed + +* Fail early when trying to add members with relative URIs in remote groups. [#5025](https://github.com/TileDB-Inc/TileDB/pull/5025) +* Correct defective return value in `Posix::ls_with_sizes`. [#5037](https://github.com/TileDB-Inc/TileDB/pull/5037) +* Prevent constructing attribute with invalid cell_val_num. [#4952](https://github.com/TileDB-Inc/TileDB/pull/4952) +* Do not mask failures when listing a directory fails on POSIX. [#5043](https://github.com/TileDB-Inc/TileDB/pull/5043) +* Fix write queries using `sm.var_offsets.extra_element=true`. [#5033](https://github.com/TileDB-Inc/TileDB/pull/5033) +* Fix segfaults in WebP queries ran in parallel. [#5065](https://github.com/TileDB-Inc/TileDB/pull/5065) +* Fix exceptions with message: ```unknown exception type; no further information```. [#5080](https://github.com/TileDB-Inc/TileDB/pull/5080) +* Fix check for out of bounds dimension in Dimension::dimension_ptr. [#5094](https://github.com/TileDB-Inc/TileDB/pull/5094) +* Fix array latest schema selection for same MS timestamps schemas. [#5143](https://github.com/TileDB-Inc/TileDB/pull/5143) +* Fix serialization issue with schema evolution for query v3. [#5154](https://github.com/TileDB-Inc/TileDB/pull/5154) + +## Configuration changes + +* Add `vfs.s3.storage_class` config option to set the storage class of newly uploaded S3 objects. [#5053](https://github.com/TileDB-Inc/TileDB/pull/5053) +* Add `rest.custom_headers.*` config option to set custom headers on REST requests. [#5104](https://github.com/TileDB-Inc/TileDB/pull/5104) +* Add `rest.payer_namespace` config option to set the namespace to be charged for REST requests. [#5105](https://github.com/TileDB-Inc/TileDB/pull/5105) + +## API changes + +### C API + +* Add CurrentDomain API support. [#5041](https://github.com/TileDB-Inc/TileDB/pull/5041) + +### C++ API + +* Current Domain CPP API implementation. [#5056](https://github.com/TileDB-Inc/TileDB/pull/5056) + +## Build System Changes + +* Backwards compatibility with older CMake versions for libfaketime. [#5049](https://github.com/TileDB-Inc/TileDB/pull/5049) +* Automatic downloading of vcpkg can be disabled by enabling the `TILEDB_DISABLE_AUTO_VCPKG` CMake option, in addition to setting the environment variable with trhe same name. [#5048](https://github.com/TileDB-Inc/TileDB/pull/5048) +* Improve embedding of `magic.mgc` and allow compiling with any libmagic version. [#4989](https://github.com/TileDB-Inc/TileDB/pull/4989) + +## Internal Improvements + +* Implement actualize function that orders data underlying `alt_var_length_view` [#5087](https://github.com/TileDB-Inc/TileDB/pull/5087) +* Implement a partitioning function to partition cells to fit into fixed size bins [#5092](https://github.com/TileDB-Inc/TileDB/pull/5092) +* Implementation of a `chunk_view` class to provide a subset of C++23 chunk_view, suitable for supporting external sort. [#5035](https://github.com/TileDB-Inc/TileDB/pull/5035) +* Tests that the chunks in a `chunk_view` can be separately sorted. [#5052](https://github.com/TileDB-Inc/TileDB/pull/5052) + # TileDB v2.24.2 Release Notes ## Defects removed diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index f9cd9d186fb1..6da80c2b146c 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -36,7 +36,7 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) # Inspired from https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.10.3/cmake-modules/AzureVcpkg.cmake message("TILEDB_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.") # To help with resolving conflicts, when you update the commit, also update its date. - set(VCPKG_COMMIT_STRING 72010900b7cee36cea77aebb97695095c9358eaf) # 2023-12-05 + set(VCPKG_COMMIT_STRING 7aeffc91033ad35cc4e2c152f213a866ec6c11ac) # 2024-07-13 message("Vcpkg commit string used: ${VCPKG_COMMIT_STRING}") include(FetchContent) FetchContent_Declare( diff --git a/examples/c_api/current_domain.c b/examples/c_api/current_domain.c index 8a626389841b..fc6fcd044847 100644 --- a/examples/c_api/current_domain.c +++ b/examples/c_api/current_domain.c @@ -74,7 +74,7 @@ void create_array() { tiledb_ndrectangle_set_range_for_name(ctx, ndrect, "d1", &range); // Assign the rectangle to the current domain - tiledb_current_domain_set_ndrectangle(current_domain, ndrect); + tiledb_current_domain_set_ndrectangle(ctx, current_domain, ndrect); // Create a single attribute "a" so each cell can store an integer tiledb_attribute_t* a; @@ -119,21 +119,21 @@ void print_current_domain() { // Check if current domain is empty uint32_t is_empty; - tiledb_current_domain_get_is_empty(current_domain, &is_empty); + tiledb_current_domain_get_is_empty(ctx, current_domain, &is_empty); if (is_empty) { printf("Current domain: empty\n"); } else { // Get current domain type tiledb_current_domain_type_t current_domain_type; - tiledb_current_domain_get_type(current_domain, ¤t_domain_type); + tiledb_current_domain_get_type(ctx, current_domain, ¤t_domain_type); if (current_domain_type == TILEDB_NDRECTANGLE) { printf("Current domain type: NDRECTANGLE\n"); // Get the ND rectangle tiledb_ndrectangle_t* ndrect; - tiledb_current_domain_get_ndrectangle(current_domain, &ndrect); + tiledb_current_domain_get_ndrectangle(ctx, current_domain, &ndrect); tiledb_range_t range; tiledb_ndrectangle_get_range_from_name(ctx, ndrect, "d1", &range); @@ -143,6 +143,23 @@ void print_current_domain() { *(int*)range.min, *(int*)range.max); + // Get datatype of range + tiledb_datatype_t dtype; + tiledb_ndrectangle_get_dtype(ctx, ndrect, 0, &dtype); + const char* dtype_str; + tiledb_datatype_to_str(dtype, &dtype_str); + printf("Range 0 dtype: %s\n", dtype_str); + + // Get datatype of range by name + tiledb_ndrectangle_get_dtype_from_name(ctx, ndrect, "d1", &dtype); + tiledb_datatype_to_str(dtype, &dtype_str); + printf("Range 0 dtype by name: %s\n", dtype_str); + + // Get dim num + uint32_t ndim; + tiledb_ndrectangle_get_dim_num(ctx, ndrect, &ndim); + printf("Range 0 dtype by name: %d\n", ndim); + // Clean up tiledb_ndrectangle_free(&ndrect); } else { @@ -191,7 +208,7 @@ void expand_current_domain() { tiledb_ndrectangle_set_range_for_name(ctx, ndrect, "d1", &range); // Set the rectangle to the current domain - tiledb_current_domain_set_ndrectangle(new_current_domain, ndrect); + tiledb_current_domain_set_ndrectangle(ctx, new_current_domain, ndrect); // Expand the current domain tiledb_array_schema_evolution_expand_current_domain( diff --git a/examples/cpp_api/current_domain.cc b/examples/cpp_api/current_domain.cc index 0ba8f3373101..3ba4be14a75e 100644 --- a/examples/cpp_api/current_domain.cc +++ b/examples/cpp_api/current_domain.cc @@ -113,6 +113,17 @@ void print_current_domain(Context& ctx) { // Print the range std::cout << "Current domain range: [" << range[0] << ", " << range[1] << "]" << std::endl; + + // Print datatype of range 0 + std::cout << "Current domain range 0 datatype: " + << tiledb::impl::type_to_str(ndrect.range_dtype(0)) << std::endl; + + // Print datatype of range d1 + std::cout << "Current domain range 0 datatype: " + << tiledb::impl::type_to_str(ndrect.range_dtype("d1")) << std::endl; + + // Print dim num + std::cout << "Current domain dim num: " << ndrect.dim_num() << std::endl; } void expand_current_domain(Context& ctx) { diff --git a/experimental/tiledb/common/dag/data_block/test/unit_blocks_and_ports.cc b/experimental/tiledb/common/dag/data_block/test/unit_blocks_and_ports.cc index aa4c96c98f08..9494594964d4 100644 --- a/experimental/tiledb/common/dag/data_block/test/unit_blocks_and_ports.cc +++ b/experimental/tiledb/common/dag/data_block/test/unit_blocks_and_ports.cc @@ -32,6 +32,7 @@ */ #include +#include #include "unit_data_block.h" diff --git a/experimental/tiledb/common/dag/data_block/test/unit_data_block.cc b/experimental/tiledb/common/dag/data_block/test/unit_data_block.cc index 35f70a2f51bf..a9b398b7a93b 100644 --- a/experimental/tiledb/common/dag/data_block/test/unit_data_block.cc +++ b/experimental/tiledb/common/dag/data_block/test/unit_data_block.cc @@ -42,6 +42,7 @@ #include #include #include +#include /** * @todo Use proper checks for preprocessor directives to selectively include diff --git a/experimental/tiledb/common/dag/edge/test/unit_edge.cc b/experimental/tiledb/common/dag/edge/test/unit_edge.cc index e8ea4c223817..ff2c53d45177 100644 --- a/experimental/tiledb/common/dag/edge/test/unit_edge.cc +++ b/experimental/tiledb/common/dag/edge/test/unit_edge.cc @@ -41,6 +41,7 @@ #include "unit_edge.h" #include +#include #include "experimental/tiledb/common/dag/edge/edge.h" #include "experimental/tiledb/common/dag/ports/ports.h" diff --git a/experimental/tiledb/common/dag/execution/test/unit_duffs.cc b/experimental/tiledb/common/dag/execution/test/unit_duffs.cc index bc8d8d1873d4..3bde011aa903 100644 --- a/experimental/tiledb/common/dag/execution/test/unit_duffs.cc +++ b/experimental/tiledb/common/dag/execution/test/unit_duffs.cc @@ -29,10 +29,12 @@ * */ -#include "unit_duffs.h" +#include + #include "../duffs.h" #include "experimental/tiledb/common/dag/edge/edge.h" #include "experimental/tiledb/common/dag/nodes/segmented_nodes.h" +#include "unit_duffs.h" using namespace tiledb::common; diff --git a/experimental/tiledb/common/dag/execution/test/unit_frugal.cc b/experimental/tiledb/common/dag/execution/test/unit_frugal.cc index 8964ccff5f21..8737c18c97c3 100644 --- a/experimental/tiledb/common/dag/execution/test/unit_frugal.cc +++ b/experimental/tiledb/common/dag/execution/test/unit_frugal.cc @@ -36,6 +36,7 @@ #include #include #include +#include #include #include diff --git a/experimental/tiledb/common/dag/execution/test/unit_scheduler.cc b/experimental/tiledb/common/dag/execution/test/unit_scheduler.cc index 777bb018c84e..547fe7b403c2 100644 --- a/experimental/tiledb/common/dag/execution/test/unit_scheduler.cc +++ b/experimental/tiledb/common/dag/execution/test/unit_scheduler.cc @@ -36,6 +36,7 @@ #include #include #include +#include #include #include "unit_scheduler.h" diff --git a/experimental/tiledb/common/dag/graph/test/unit_taskgraph.cc b/experimental/tiledb/common/dag/graph/test/unit_taskgraph.cc index 11838ddde19a..36938c40fb64 100644 --- a/experimental/tiledb/common/dag/graph/test/unit_taskgraph.cc +++ b/experimental/tiledb/common/dag/graph/test/unit_taskgraph.cc @@ -28,10 +28,12 @@ * @section DESCRIPTION */ -#include "unit_taskgraph.h" +#include + #include "experimental/tiledb/common/dag/execution/duffs.h" #include "experimental/tiledb/common/dag/graph/taskgraph.h" #include "experimental/tiledb/common/dag/nodes/segmented_nodes.h" +#include "unit_taskgraph.h" using namespace tiledb::common; diff --git a/experimental/tiledb/common/dag/nodes/generators.h b/experimental/tiledb/common/dag/nodes/generators.h index 8095b0f638c5..ee766682210f 100644 --- a/experimental/tiledb/common/dag/nodes/generators.h +++ b/experimental/tiledb/common/dag/nodes/generators.h @@ -33,6 +33,7 @@ #ifndef TILEDB_DAG_GENERATOR_H #define TILEDB_DAG_GENERATOR_H +#include #include #include "experimental/tiledb/common/dag/ports/ports.h" diff --git a/experimental/tiledb/common/dag/nodes/test/unit_general_nodes.cc b/experimental/tiledb/common/dag/nodes/test/unit_general_nodes.cc index fe5230136b17..bb7607067e1b 100644 --- a/experimental/tiledb/common/dag/nodes/test/unit_general_nodes.cc +++ b/experimental/tiledb/common/dag/nodes/test/unit_general_nodes.cc @@ -32,6 +32,7 @@ #include "unit_general_nodes.h" #include +#include #include "experimental/tiledb/common/dag/edge/edge.h" #include "experimental/tiledb/common/dag/nodes/detail/simple/mimo.h" diff --git a/experimental/tiledb/common/dag/nodes/test/unit_segmented_mimo_nodes.cc b/experimental/tiledb/common/dag/nodes/test/unit_segmented_mimo_nodes.cc index 8b956cae2306..3d1b30de0c3c 100644 --- a/experimental/tiledb/common/dag/nodes/test/unit_segmented_mimo_nodes.cc +++ b/experimental/tiledb/common/dag/nodes/test/unit_segmented_mimo_nodes.cc @@ -32,6 +32,8 @@ * @todo: Need to get better syntax for Edge with shared_ptr */ +#include + #include "unit_segmented_mimo_nodes.h" #include "experimental/tiledb/common/dag/edge/edge.h" diff --git a/experimental/tiledb/common/dag/nodes/test/unit_simple_nodes.cc b/experimental/tiledb/common/dag/nodes/test/unit_simple_nodes.cc index cb8af6f018c1..6329b301fed9 100644 --- a/experimental/tiledb/common/dag/nodes/test/unit_simple_nodes.cc +++ b/experimental/tiledb/common/dag/nodes/test/unit_simple_nodes.cc @@ -32,6 +32,7 @@ #include "unit_simple_nodes.h" #include +#include #include #include "experimental/tiledb/common/dag/edge/edge.h" #include "experimental/tiledb/common/dag/nodes/generators.h" diff --git a/experimental/tiledb/common/dag/nodes/test/unit_stop_source.cc b/experimental/tiledb/common/dag/nodes/test/unit_stop_source.cc index 14c273a867f0..98110f7a70c5 100644 --- a/experimental/tiledb/common/dag/nodes/test/unit_stop_source.cc +++ b/experimental/tiledb/common/dag/nodes/test/unit_stop_source.cc @@ -32,6 +32,7 @@ #include #include +#include #include "unit_stop_source.h" diff --git a/experimental/tiledb/common/dag/nodes/test/unit_util_functions.cc b/experimental/tiledb/common/dag/nodes/test/unit_util_functions.cc index 7ccf39b665d2..6ed9b330dbb7 100644 --- a/experimental/tiledb/common/dag/nodes/test/unit_util_functions.cc +++ b/experimental/tiledb/common/dag/nodes/test/unit_util_functions.cc @@ -33,6 +33,7 @@ #include "unit_util_functions.h" #include +#include #include "experimental/tiledb/common/dag/edge/edge.h" #include "experimental/tiledb/common/dag/nodes/generators.h" #include "experimental/tiledb/common/dag/nodes/nodes.h" diff --git a/experimental/tiledb/common/dag/ports/test/unit_ports.cc b/experimental/tiledb/common/dag/ports/test/unit_ports.cc index cbb958c6abda..002163459e23 100644 --- a/experimental/tiledb/common/dag/ports/test/unit_ports.cc +++ b/experimental/tiledb/common/dag/ports/test/unit_ports.cc @@ -32,6 +32,7 @@ #include "unit_ports.h" #include +#include #include "experimental/tiledb/common/dag/ports/ports.h" #include "experimental/tiledb/common/dag/state_machine/policies.h" #include "experimental/tiledb/common/dag/state_machine/test/types.h" diff --git a/experimental/tiledb/common/dag/utility/test/unit_bounded_buffer.cc b/experimental/tiledb/common/dag/utility/test/unit_bounded_buffer.cc index 21fbeecda3e9..3bc2d12669e0 100644 --- a/experimental/tiledb/common/dag/utility/test/unit_bounded_buffer.cc +++ b/experimental/tiledb/common/dag/utility/test/unit_bounded_buffer.cc @@ -32,6 +32,7 @@ #include "experimental/tiledb/common/dag/utility/test/unit_bounded_buffer.h" #include +#include #include #include #include "experimental/tiledb/common/dag/utility/bounded_buffer.h" diff --git a/format_spec/FORMAT_SPEC.md b/format_spec/FORMAT_SPEC.md index e7750a9b9e23..77834612d898 100644 --- a/format_spec/FORMAT_SPEC.md +++ b/format_spec/FORMAT_SPEC.md @@ -11,36 +11,10 @@ title: Format Specification - [Dictionary filter](filters/dictionary_encoding.md) - RLE filter -## History - -|Format version|TileDB version|Description| -|-|-|-| -|1|1.4|[Decouple format and library version](https://github.com/TileDB-Inc/TileDB/commit/610f087515b6de5c3290b09dab30c6943ec77feb)| -|2|1.5|[Always split coordinate tiles](https://github.com/TileDB-Inc/TileDB/commit/9394b38bdfbacd606d673896b4ae87e7968b7c2f)| -|3|1.6|[Parallelize fragment metadata loading](https://github.com/TileDB-Inc/TileDB/commit/a2eb6237e622c3a17691dbe04c9223ba099f7466)| -|4|1.7|[Remove KV storage](https://github.com/TileDB-Inc/TileDB/commit/e733f7baa85a41e25e5834a220234397d6038401)| -|5|2.0|[Split coordinates into individual files](https://github.com/TileDB-Inc/TileDB/commit/d3543bdbc4ee7c2ed1f2de8cee42b04c6ec8eafc)| -|6|2.1|[Implement attribute fill values](https://github.com/TileDB-Inc/TileDB/commit/eaafa47c97af0ee654a0ca2e97da7b8d941e672b)| -|7|2.2|[Nullable attribute support](https://github.com/TileDB-Inc/TileDB/commit/a7fd8d6dd74bb4fa1ae25a6f995da93812f92c20)| -|8|2.3|[Percent encode attribute/dimension file names](https://github.com/TileDB-Inc/TileDB/commit/97c5c4b0aa35cfd96197558ffc1189860b4adc6f)| -|9|2.3|[Name attribute/dimension files by index](https://github.com/TileDB-Inc/TileDB/commit/9a2ed1c22242f097300c2909baf6cb671a7ee33e)| -|10|2.4|[Added array schema evolution](https://github.com/TileDB-Inc/TileDB/commit/41e5e8f4b185f49777560d637b1d61de498364ce)| -|11|2.7|[Store integral cells, aka, don't split cells across chunks](https://github.com/TileDB-Inc/TileDB/commit/beab5113526b7156c8c6492542f1681555c8ae87)| -|12|2.8|[New array directory structure](https://github.com/TileDB-Inc/TileDB/commit/ce204ad1ea5b40f006f4a6ddf240d89c08b3235b)| -|13|2.9|[Add dictionary filter](https://github.com/TileDB-Inc/TileDB/commit/5637e8c678451c9d2356ccada118b504c8ca85f0)| -|14|2.10|[Consolidation with timestamps, add has_timestamps to footer](https://github.com/TileDB-Inc/TileDB/commit/31a3dce8db254efc36f6d28249febed41bba3bcd)| -|15|2.11|[Remove consolidate with timestamps config](https://github.com/TileDB-Inc/TileDB/commit/6b49739e79d804dc56eb0a7e422823ae6f002276)| -|16|2.12|[Implement delete strategy](https://github.com/TileDB-Inc/TileDB/commit/8d64b1f38177113379fa741016136dbd2b06fcfd)| -|17|2.14|[Add dimension labels and data order](https://github.com/TileDB-Inc/TileDB/commit/bb433fcf12dc74a38c7e843808ec1e593b16ce71)| -|18|2.15|[Dimension Labels no longer experimental](https://github.com/TileDB-Inc/TileDB/commit/c3a1bb47e7237f50e8ed9e33abfaa3161e23ff64)| -|19|2.16|[Vac files now use relative URIs](https://github.com/TileDB-Inc/TileDB/commit/ef3236a526b67c50138436a16f67ad274c2ca037)| -|20|2.17|[Enumerations](https://github.com/TileDB-Inc/TileDB/commit/c0d7c6a50fdeffbcc7d8c9ba4a29230fe22baed6)| -|21|2.19|[Tile metadata are now correctly calculated for nullable fixed size strings on dense arrays](https://github.com/TileDB-Inc/TileDB/commit/081bcc5f7ce4bee576f08b97de348236ac88d429)| -|22|2.25|[Add array current domain](https://github.com/TileDB-Inc/TileDB/commit/9116d3c95a83d72545520acb9a7808fc63478963)| - ## Table of Contents * **Array** + * [Format Version History](./history.md) * [File hierarchy](./array_file_hierarchy.md) * [Array Schema](./array_schema.md) * [Fragment](./fragment.md) @@ -53,4 +27,4 @@ title: Format Specification * [Consolidated Fragment Metadata File](./consolidated_fragment_metadata_file.md) * [Filter Pipeline](./filter_pipeline.md) * [Timestamped Name](./timestamped_name.md) - * [Vacuum Pipeline](./vacuum_file.md) + * [Vacuum File](./vacuum_file.md) diff --git a/format_spec/history.md b/format_spec/history.md new file mode 100644 index 000000000000..edf5c8400e79 --- /dev/null +++ b/format_spec/history.md @@ -0,0 +1,169 @@ +--- +title: Format version history +--- + +# Format Version History + +## Version 22 + +Introduced in TileDB 2.25 + +* The _Current domain_ field was added to [array schemas](./array_schema.md#array-schema-file). + +## Version 21 + +Introduced in TileDB 2.19 + +* The TileDB implementation has been updated to fix computing [tile metadata](./fragment.md#tile-mins-maxes) for nullable fixed-size strings on dense arrays. + +> [!NOTE] +> This version does not contain any changes to the storage format, but was introduced as an indicator for implementations to not rely on tile metadata for nullable fixed-size strings on dense arrays on previous versions. + +## Version 20 + +Introduced in TileDB 2.17 + +* Arrays can have [enumerations](./enumeration.md). +* The bit-width reduction and positive delta filters are supported on data of date or time types. +* The [filter pipeline options](./filter_pipeline.md#filter-options) for the double-delta filter contain the _Reinterpret datatype_ field. + +## Version 19 + +Introduced in TileDB 2.16 + +* [Vacuum files](./vacuum_file.md) contain relative paths to the location of the array. +* The [filter pipeline options](./filter_pipeline.md#filter-options) for the delta filter contain the _Reinterpret datatype_ field. + +## Version 18 + +Introduced in TileDB 2.15 + +* Arrays can have [dimension labels](./array_schema.md#dimension-label). + +## Version 17 + +Introduced in TileDB 2.14 + +* The _Order_ field was added to [attributes](./array_schema.md#attribute). +* Cell offsets in dimensions or attributes of UTF-8 string type are not written in the offset tiles, if the RLE or dictionary filter exists in the filter pipeline. They are instead encoded as part of the data tile. + +## Version 16 + +Introduced in TileDB 2.12 + +* Arrays can have [delete commit files](./delete_commit_file.md). +* Arrays can have [update commit files](./update_commit_file.md). + * The TileDB implementation currently supports writing update commit files as an experimental feature, but they are not yet considered when performing reads. +* Fragment metadata contain [tile processed conditions](./fragment.md#tile-processed-conditions). + +## Version 15 + +Introduced in TileDB 2.11 + +* Consolidated fragments can have delete metadata files. The _Includes delete metadata_ field was added to the [fragment metadata footer](./fragment.md#footer). + +## Version 14 + +Introduced in TileDB 2.10 + +* Consolidated fragments can have timestamp files. The _Includes timestamps_ field was added to the [fragment metadata footer](./fragment.md#footer). + +## Version 13 + +Introduced in TileDB 2.9 + +* The [dictionary filter](./filters/dictionary_encoding.md) was added. + +## Version 12 + +Introduced in TileDB 2.8 + +* The [array file hierarchy](./array_file_hierarchy.md) was updated to store fragments, commits and consolidated fragment metadata in separate subdirectories. +* The extension of commit files was changed to `.wrt`. +* Cell offsets in dimensions or attributes of ASCII string type are not written in the offset tiles, if the RLE filter exists in the filter pipeline. They are instead encoded as part of the data tile. + +## Version 11 + +Introduced in TileDB 2.7 + +* Fragment metadata contain [metadata](./fragment.md#tile-mins-maxes) (min/max value, sum, null count) for each tile. +* The TileDB implementation has been updated to never split cells when storing them in chunks. + +## Version 10 + +Introduced in TileDB 2.4 + +* Arrays support schema evolution. + * Array schemas are stored in a `__schema` subdirectory, and have a [timestamped name](./timestamped_name.md). + * The _Array schema name_ field was added to the [fragment metadata footer](./fragment.md#footer). +* The _Footer length_ field of the [fragment metadata footer](./fragment.md#footer) is always written. + +## Version 9 + +Introduced in TileDB 2.3 + +* [Data files](./fragment.md#data-file) are named by the index of their attribute or dimension. +* The _URI_ fields of [Consolidated fragment metadata files](./consolidated_fragment_metadata_file.md) contain relative paths to the location of fragments in the array. + +## Version 8 + +Introduced in TileDB 2.2.3 + +* [Data files](./fragment.md#data-file) are named by the name of their attribute or dimension, after percent encoding certain characters. These characters are `!#$%&'()*+,/:;=?@[]`, as specified in [RFC 3986](https://tools.ietf.org/html/rfc3986), as well as `"<>\|`, which are not allowed in Windows file names. + +## Version 7 + +Introduced in TileDB 2.2 + +* Attributes can be nullable. + * The _Nullable_ and _Fill value validity_ fields were added to [attributes](./array_schema.md#attribute). + * The _Validity filters_ field was added to [array schemas](./array_schema.md#array-schema-file). + * Fragment metadata contain validity [tile offsets](./fragment.md#tile-offsets). + +## Version 6 + +Introduced in TileDB 2.1 + +* The _Fill value_ field was added to [attributes](./array_schema.md#attribute). + +## Version 5 + +Introduced in TileDB 2.0 + +* Dimensions are stored in separate [data files](./fragment.md#data-file). +* Sparse arrays can have string dimensions and dimensions with different datatypes. + * The _Dimension datatype_, _Cell val num_ and _Filters_ fields were added to [dimensions](./array_schema.md#dimension). + * The _Domain size_ field was added to [dimensions](./array_schema.md#dimension). The domain of a dimension can have a variable size. + * The _Domain datatype_ field was removed from [domains](./array_schema.md#domain). + * The [MBR](./fragment.md#mbr) structure has been updated to support variable-sized dimensions. + * The _Dimension number_ and _R-Tree datatype_ fields have been removed from [R-Trees](./fragment.md#r-tree). +* The _Allows dups_ field was added to [array schemas](./array_schema.md#array-schema-file). +* Committed fragments are indicated by the presence of an `.ok` file in the array's directory, with the same [timestamped name](./timestamped_name.md) as the fragment. + +## Version 4 + +Introduced in TileDB 1.7 + +* Support for the [key-value store](https://tiledb-inc-tiledb.readthedocs-hosted.com/en/1.6.3/tutorials/kv.html) object type was removed. Key-value stores have been superseded by sparse arrays. + +## Version 3 + +Introduced in TileDB 1.6 + +* The structure of [fragment metadata files](./fragment.md#fragment-metadata-file) was overhauled. + * The [footer](./fragment.md#footer) and [R-Tree](./fragment.md#r-tree) structures were added. + * The _Bounding coords_ field was removed. + * The _MBRs_ field was removed. MBRs are now stored in the R-Tree. + * Structures other than the footer like tile offsets, sizes and metadata are wrapped in their own generic tiles. This allows loading them lazily and in parallel. + +## Version 2 + +Introduced in TileDB 1.5 + +* Cell coordinate values of each dimension are always stored next to each other, regardless of whether they are filtered with a compression filter or not. + +## Version 1 + +Introduced in TileDB 1.4 + +* Initial version of the TileDB storage format. diff --git a/format_spec/vacuum_file.md b/format_spec/vacuum_file.md index 280e8544dedb..290c8997b8ff 100644 --- a/format_spec/vacuum_file.md +++ b/format_spec/vacuum_file.md @@ -22,7 +22,7 @@ my_array # array folder | ... ``` -When located in the commits folder, it will include the URI of fragments (in the `__fragments` folder) that can be vaccumed. When located in the array metadata folder, it will include the URI or array metadata files that can be vaccumed. +When located in the commits folder, it will include the URI of fragments (in the `__fragments` folder) that can be vacuumed. When located in the array metadata folder, it will include the URI or array metadata files that can be vacuumed. The vacuum file is a simple text file where each line contains a URI string: diff --git a/ports/README.md b/ports/README.md index 299ed29fb802..26e98d4e5020 100644 --- a/ports/README.md +++ b/ports/README.md @@ -26,13 +26,13 @@ $ git commit After copying the port, add an entry to the table below. You should also contribute your changes to vcpkg and/or the upstream package repository. +For ease of review when patching existing ports, you are recommended to make one commit that copies the upstream port unchanged, and another commit that makes the changes you need. + ## List of port overlays -| Port | Reason | -|----------------------------|---------------------------------------------------------------------------------------------------------| -| `libmagic` | Updating to the upstream port deferred due to failures. | -| `pcre2` | To be removed alongside libmagic. | -| `azure-storage-common-cpp` | Patching to disable default features on libxml2 (https://github.com/Azure/azure-sdk-for-cpp/pull/5221). | -| `libfaketime` | Port does not yet exist upstream | -| `vcpkg-cmake-config` | Patching to fix build issues with CMake 3.29.1. (https://github.com/microsoft/vcpkg/pull/38017) | -| `google-cloud-cpp` | Patching to remove dependency on GMock. (https://github.com/microsoft/vcpkg/pull/39802) | +| Port | Reason | +|---------------|----------------------------------------------------------------------------------------------------------------------------| +| `aws-c-io` | Patching to fix MinGW build failures. (https://github.com/awslabs/aws-c-io/pull/664) | +| `aws-sdk-cpp` | Patching to fix MinGW build failures, and to avoid building test-only SDKs (https://github.com/aws/aws-sdk-cpp/pull/3061). | +| `libmagic` | Updating to the upstream port deferred due to failures. | +| `libfaketime` | Port does not yet exist upstream | diff --git a/ports/aws-c-io/fix-pointer-mismatch.patch b/ports/aws-c-io/fix-pointer-mismatch.patch new file mode 100644 index 000000000000..474624554e29 --- /dev/null +++ b/ports/aws-c-io/fix-pointer-mismatch.patch @@ -0,0 +1,22 @@ +diff --git a/source/windows/secure_channel_tls_handler.c b/source/windows/secure_channel_tls_handler.c +index b62b1a0..48f4556 100644 +--- a/source/windows/secure_channel_tls_handler.c ++++ b/source/windows/secure_channel_tls_handler.c +@@ -53,7 +53,7 @@ struct secure_channel_ctx { + struct aws_tls_ctx ctx; + struct aws_string *alpn_list; + SCHANNEL_CRED credentials; +- PCERT_CONTEXT pcerts; ++ PCCERT_CONTEXT pcerts; + HCERTSTORE cert_store; + HCERTSTORE custom_trust_store; + HCRYPTPROV crypto_provider; +@@ -188,7 +188,7 @@ static int s_manually_verify_peer_cert(struct aws_channel_handler *handler) { + int result = AWS_OP_ERR; + CERT_CONTEXT *peer_certificate = NULL; + HCERTCHAINENGINE engine = NULL; +- CERT_CHAIN_CONTEXT *cert_chain_ctx = NULL; ++ PCCERT_CHAIN_CONTEXT cert_chain_ctx = NULL; + + /* get the peer's certificate so we can validate it.*/ + SECURITY_STATUS status = diff --git a/ports/aws-c-io/portfile.cmake b/ports/aws-c-io/portfile.cmake new file mode 100644 index 000000000000..0d814d7398b1 --- /dev/null +++ b/ports/aws-c-io/portfile.cmake @@ -0,0 +1,34 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO awslabs/aws-c-io + REF "v${VERSION}" + SHA512 b5dc81635ce775c3783df2d4f6ccf0890b1e1669bddb1c10f6d200203684e84504ca87099cb88874010df28f98f658da5a4eacb9e4df0408d40d845f3d394f48 + HEAD_REF master + PATCHES + "fix-pointer-mismatch.patch" +) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files + -DBUILD_TESTING=FALSE +) + +vcpkg_cmake_install() + +string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") +vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) +vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") +vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") + +file(REMOVE_RECURSE + "${CURRENT_PACKAGES_DIR}/debug/include" + "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" + "${CURRENT_PACKAGES_DIR}/debug/share" + "${CURRENT_PACKAGES_DIR}/lib/${PORT}" +) + +vcpkg_copy_pdbs() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-io/vcpkg.json b/ports/aws-c-io/vcpkg.json new file mode 100644 index 000000000000..cf067314159a --- /dev/null +++ b/ports/aws-c-io/vcpkg.json @@ -0,0 +1,24 @@ +{ + "name": "aws-c-io", + "version": "0.14.9", + "description": "Handles all IO and TLS work for application protocols.", + "homepage": "https://github.com/awslabs/aws-c-io", + "license": "Apache-2.0", + "supports": "!(windows & arm) & !uwp", + "dependencies": [ + "aws-c-cal", + "aws-c-common", + { + "name": "s2n", + "platform": "!uwp & !windows" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/ports/aws-sdk-cpp/find-dependency.patch b/ports/aws-sdk-cpp/find-dependency.patch new file mode 100644 index 000000000000..4ddf5156428d --- /dev/null +++ b/ports/aws-sdk-cpp/find-dependency.patch @@ -0,0 +1,14 @@ +diff --git a/toolchains/core-config.cmake b/toolchains/core-config.cmake +index d90bfaf747..ab14402577 100644 +--- a/toolchains/core-config.cmake ++++ b/toolchains/core-config.cmake +@@ -15,5 +15,9 @@ if (AWSSDK_CRYPTO_IN_SOURCE_BUILD) + find_dependency(crypto) + find_dependency(ssl) + endif() ++find_dependency(ZLIB) ++if("@ENABLE_CURL_CLIENT@") ++ find_dependency(CURL) ++endif() + set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_PREV}) + include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@-targets.cmake") diff --git a/ports/aws-sdk-cpp/fix-aws-root.patch b/ports/aws-sdk-cpp/fix-aws-root.patch new file mode 100644 index 000000000000..da4365ad7218 --- /dev/null +++ b/ports/aws-sdk-cpp/fix-aws-root.patch @@ -0,0 +1,22 @@ +diff --git a/cmake/AWSSDKConfig.cmake b/cmake/AWSSDKConfig.cmake +--- a/cmake/AWSSDKConfig.cmake (revision 2f90f9fd6c56460bd382243aa215fcddcb5883c8) ++++ b/cmake/AWSSDKConfig.cmake (date 1636913220527) +@@ -54,18 +54,14 @@ + string(REPLACE ";" "${AWS_MODULE_DIR};" SYSTEM_MODULE_PATH "${CMAKE_SYSTEM_PREFIX_PATH}${AWS_MODULE_DIR}") + list(APPEND CMAKE_MODULE_PATH ${AWS_MODULE_PATH} ${SYSTEM_MODULE_PATH}) + +-# On Windows, dlls are treated as runtime target and installed in bindir + if (WIN32 AND AWSSDK_INSTALL_AS_SHARED_LIBS) +- set(AWSSDK_INSTALL_LIBDIR "${AWSSDK_INSTALL_BINDIR}") + # If installed CMake scripts are associated with dll library, define USE_IMPORT_EXPORT for customers + add_definitions(-DUSE_IMPORT_EXPORT) + endif() + + + # Compute the default installation root relative to this file. +-# from prefix/lib/cmake/AWSSDK/xx.cmake to prefix + get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +-get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${AWSSDK_DEFAULT_ROOT_DIR}" PATH) + get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${AWSSDK_DEFAULT_ROOT_DIR}" PATH) + get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${AWSSDK_DEFAULT_ROOT_DIR}" PATH) + get_filename_component(AWS_NATIVE_SDK_ROOT "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE) diff --git a/ports/aws-sdk-cpp/fix-winsock-headers.patch b/ports/aws-sdk-cpp/fix-winsock-headers.patch new file mode 100644 index 000000000000..d83754655ec0 --- /dev/null +++ b/ports/aws-sdk-cpp/fix-winsock-headers.patch @@ -0,0 +1,14 @@ +diff --git a/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +index b8ab519..7cd91e9 100644 +--- a/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp ++++ b/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +@@ -19,6 +19,9 @@ + + #include + #include ++#ifdef __MINGW32__ ++#include ++#endif + #include // for tcp_keepalive + #include + #include diff --git a/ports/aws-sdk-cpp/fix_find_curl.patch b/ports/aws-sdk-cpp/fix_find_curl.patch new file mode 100644 index 000000000000..b33505265417 --- /dev/null +++ b/ports/aws-sdk-cpp/fix_find_curl.patch @@ -0,0 +1,31 @@ +diff --git a/cmake/external_dependencies.cmake b/cmake/external_dependencies.cmake +index acf16c0..3a49fb4 100644 +--- a/cmake/external_dependencies.cmake ++++ b/cmake/external_dependencies.cmake +@@ -80,23 +80,12 @@ if(NOT NO_HTTP_CLIENT AND NOT USE_CRT_HTTP_CLIENT) + set(BUILD_CURL 1) + message(STATUS " Building Curl as part of AWS SDK") + else() +- include(FindCURL) ++ find_package(CURL REQUIRED) + if(NOT CURL_FOUND) + message(FATAL_ERROR "Could not find curl") ++ else() ++ set(CURL_LIBRARIES CURL::libcurl) + endif() +- +- # When built from source using cmake, curl does not include +- # CURL_INCLUDE_DIRS or CURL_INCLUDE_DIRS so we need to use +- # find_package to fix it +- if ("${CURL_INCLUDE_DIRS}" STREQUAL "" AND "${CURL_LIBRARIES}" STREQUAL "") +- message(STATUS "Could not find curl include or library path, falling back to find with config.") +- find_package(CURL) +- set(CURL_LIBRARIES CURL::libcurl) +- else () +- message(STATUS " Curl include directory: ${CURL_INCLUDE_DIRS}") +- List(APPEND EXTERNAL_DEPS_INCLUDE_DIRS ${CURL_INCLUDE_DIRS}) +- set(CLIENT_LIBS ${CURL_LIBRARIES}) +- endif () + set(CLIENT_LIBS_ABSTRACT_NAME curl) + message(STATUS " Curl target link: ${CURL_LIBRARIES}") + endif() diff --git a/ports/aws-sdk-cpp/generateFeatures.ps1 b/ports/aws-sdk-cpp/generateFeatures.ps1 new file mode 100644 index 000000000000..299ef13583d6 --- /dev/null +++ b/ports/aws-sdk-cpp/generateFeatures.ps1 @@ -0,0 +1,73 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory=$true)][string]$SourcesRef, + [Parameter(Mandatory=$false)][string]$PortDirectory = $PSScriptRoot, + [Parameter(Mandatory=$false)][string]$vcpkg = "$PSScriptRoot/../../vcpkg" +) + +$ErrorActionPreference = "Stop" + +$ManifestIn = "$PortDirectory/vcpkg.in.json" +$ManifestOut = "$PortDirectory/vcpkg.json" + +$ExtractedSources = "${env:TEMP}/aws-sdk-cpp-generateFeatures-$SourcesRef" +if (-not (Test-Path $ExtractedSources)) { + if (Test-Path "$ExtractedSources.tmp") { + Remove-Item -Force "$ExtractedSources.tmp" + } + git clone --depth=1 "https://github.com/aws/aws-sdk-cpp" "$ExtractedSources.tmp" | Out-Host + git -c "$ExtractedSources.tmp" checkout $SourcesRef + Move-Item "$ExtractedSources.tmp" "$ExtractedSources" +} +Write-Host "Using sources directory: $ExtractedSources" + + +$subfolders = Get-ChildItem -Path "$ExtractedSources\generated\src\aws-cpp-sdk-*", "$ExtractedSources\src\aws-cpp-sdk*" | Sort-Object -Property Name + +$manifest = Get-Content $ManifestIn | ConvertFrom-Json +$manifest | Add-Member ` + -NotePropertyName '$note' ` + -NotePropertyValue 'Automatically generated by generateFeatures.ps1' +$manifest | Add-Member -NotePropertyName 'features' -NotePropertyValue @{} + +function GetDescription($dir, $modulename) +{ + if (Test-Path "$dir\CMakeLists.txt") + { + $descs = @(Select-String -Path "$dir\CMakeLists.txt" -Pattern "`"C\+\+ SDK for the AWS [^`"]*`"") + if ($descs.count -eq 1) { + $desc = $descs[0].Matches.Value -replace "`"","" + "$desc" + } + else { "C++ SDK for the AWS $modulename service" } + } + else { "C++ SDK for the AWS $modulename service" } +} + +$featureDependencies = @{} +Select-String -Path "$ExtractedSources\cmake\sdksCommon.cmake" -Pattern "list\(APPEND SDK_DEPENDENCY_LIST `"([\w-]+):([\w-,]+)`"\)" -AllMatches ` +| ForEach-Object { $_.Matches } ` +| ForEach-Object { $featureDependencies[$_.Groups[1].Value] = @($_.Groups[2].Value -split "," ` +| Where-Object { $_ -ne "core" }) } + +foreach ($subfolder in $subfolders) +{ + $modulename = $subfolder.name -replace "^aws-cpp-sdk-","" + if ($modulename -match "-tests`$") { continue } + if ($modulename -match "-sample`$") { continue } + if ($modulename -eq "core") { continue } + + $lowermodulename = $modulename.ToLower() + + $featureObj = @{ description = (GetDescription $subfolder $modulename) } + + if ($featureDependencies.ContainsKey($lowermodulename)) { + $featureObj.dependencies = ,@{ name = "aws-sdk-cpp"; "default-features" = $false; "features" = $featureDependencies[$lowermodulename] } + } + + $manifest.features.Add("$lowermodulename", $featureObj) +} + +[IO.File]::WriteAllText($ManifestOut, (ConvertTo-Json -Depth 10 -InputObject $manifest)) + +& $vcpkg format-manifest --feature-flags=-manifests $ManifestOut diff --git a/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch b/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch new file mode 100644 index 000000000000..0f7a3b191830 --- /dev/null +++ b/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch @@ -0,0 +1,20 @@ +diff --git a/src/aws-cpp-sdk-core/CMakeLists.txt b/src/aws-cpp-sdk-core/CMakeLists.txt +index c44546b0e..b66888362 100644 +--- a/src/aws-cpp-sdk-core/CMakeLists.txt ++++ b/src/aws-cpp-sdk-core/CMakeLists.txt +@@ -113,13 +113,8 @@ if(ENABLE_CURL_CLIENT) + int main() { + CURL* handle = curl_easy_init(); + return curl_easy_setopt(handle, CURLOPT_PROXY_SSLCERT, \"client.pem\"); }") +- if (CMAKE_CROSSCOMPILING) +- check_c_source_compiles("${CHECK_CURL_HAS_H2}" CURL_HAS_H2) +- check_c_source_compiles("${CHECK_CURL_HAS_TLS_PROXY}" CURL_HAS_TLS_PROXY) +- else() +- check_c_source_runs("${CHECK_CURL_HAS_H2}" CURL_HAS_H2) +- check_c_source_runs("${CHECK_CURL_HAS_TLS_PROXY}" CURL_HAS_TLS_PROXY) +- endif() ++ set(CURL_HAS_H2 OFF) ++ set(CURL_HAS_TLS_PROXY ON) + elseif(ENABLE_WINDOWS_CLIENT) + # NOTE: HTTP/2 is not supported when using IXML_HTTP_REQUEST_2 + if(USE_IXML_HTTP_REQUEST_2) diff --git a/ports/aws-sdk-cpp/no-test-deps.patch b/ports/aws-sdk-cpp/no-test-deps.patch new file mode 100644 index 000000000000..167bccbb5ffc --- /dev/null +++ b/ports/aws-sdk-cpp/no-test-deps.patch @@ -0,0 +1,25 @@ +diff --git a/cmake/sdks.cmake b/cmake/sdks.cmake +index 75283cdd1b6..71d383e3323 100644 +--- a/cmake/sdks.cmake ++++ b/cmake/sdks.cmake +@@ -34,12 +34,14 @@ if(BUILD_ONLY) + endforeach() + endif() + +- get_dependencies_for_test(${TARGET} DEPENDENCY_LIST) +- if(DEPENDENCY_LIST) +- STRING(REPLACE "," ";" LIST_RESULT ${DEPENDENCY_LIST}) +- foreach(DEPENDENCY IN LISTS LIST_RESULT) +- list(APPEND SDK_DEPENDENCY_BUILD_LIST ${DEPENDENCY}) +- endforeach() ++ if(ENABLE_TESTING) ++ get_dependencies_for_test(${TARGET} DEPENDENCY_LIST) ++ if(DEPENDENCY_LIST) ++ STRING(REPLACE "," ";" LIST_RESULT ${DEPENDENCY_LIST}) ++ foreach(DEPENDENCY IN LISTS LIST_RESULT) ++ list(APPEND SDK_DEPENDENCY_BUILD_LIST ${DEPENDENCY}) ++ endforeach() ++ endif() + endif() + endforeach() + LIST(REMOVE_DUPLICATES SDK_BUILD_LIST) diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake new file mode 100644 index 000000000000..05d281781b4f --- /dev/null +++ b/ports/aws-sdk-cpp/portfile.cmake @@ -0,0 +1,104 @@ +vcpkg_buildpath_length_warning(37) + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO aws/aws-sdk-cpp + REF "${VERSION}" + SHA512 cdf2878d573398d65a20ce65d994552489ec06630e7db7db561f2e662d052d77c75395efc23445a610150817ac7db3eaaa66a42f4af6c7cee76216b04ce63dc7 + PATCHES + fix-aws-root.patch + lock-curl-http-and-tls-settings.patch + fix_find_curl.patch + find-dependency.patch + fix-winsock-headers.patch # In MinGW, mstcpip.h references types from headers it does not include itself. + no-test-deps.patch # https://github.com/aws/aws-sdk-cpp/pull/3061 +) + +string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT) + +set(EXTRA_ARGS "") +if(VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) + set(rpath "@loader_path") +elseif (VCPKG_TARGET_IS_ANDROID) + set(EXTRA_ARGS "-DTARGET_ARCH=ANDROID" + "-DGIT_EXECUTABLE=--invalid-git-executable--" + "-DGIT_FOUND=TRUE" + "-DNDK_DIR=$ENV{ANDROID_NDK_HOME}" + "-DANDROID_BUILD_ZLIB=FALSE" + "-DANDROID_BUILD_CURL=FALSE" + "-DANDROID_BUILD_OPENSSL=FALSE" + ) +else() + set(rpath "\$ORIGIN") +endif() + +string(REPLACE "awsmigrationhub" "AWSMigrationHub" targets "${FEATURES}") +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + ${EXTRA_ARGS} + "-DENABLE_UNITY_BUILD=ON" + "-DENABLE_TESTING=OFF" + "-DFORCE_SHARED_CRT=${FORCE_SHARED_CRT}" + "-DBUILD_ONLY=${targets}" + "-DBUILD_DEPS=OFF" + "-DBUILD_SHARED_LIBS=OFF" + "-DAWS_SDK_WARNINGS_ARE_ERRORS=OFF" + "-DCMAKE_INSTALL_RPATH=${rpath}" + "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files +) +vcpkg_cmake_install() + +foreach(TARGET IN LISTS targets) + string(TOLOWER "aws-cpp-sdk-${TARGET}" package) + vcpkg_cmake_config_fixup(PACKAGE_NAME "${package}" CONFIG_PATH "lib/cmake/aws-cpp-sdk-${TARGET}" DO_NOT_DELETE_PARENT_CONFIG_PATH) +endforeach() +vcpkg_cmake_config_fixup(PACKAGE_NAME "awssdk" CONFIG_PATH "lib/cmake/AWSSDK") + +vcpkg_copy_pdbs() + +file(GLOB_RECURSE AWS_TARGETS "${CURRENT_PACKAGES_DIR}/share/*/*-targets-*.cmake") +foreach(AWS_TARGET IN LISTS AWS_TARGETS) + file(READ ${AWS_TARGET} _contents) + string(REGEX REPLACE + "bin\\/([A-Za-z0-9_.-]+\\.lib)" + "lib/\\1" + _contents "${_contents}") + file(WRITE ${AWS_TARGET} "${_contents}") +endforeach() + +file(GLOB AWS_CONFIGS "${CURRENT_PACKAGES_DIR}/share/*/aws-cpp-sdk-*-config.cmake") +list(FILTER AWS_CONFIGS EXCLUDE REGEX "aws-cpp-sdk-core-config\\.cmake\$") +foreach(AWS_CONFIG IN LISTS AWS_CONFIGS) + file(READ "${AWS_CONFIG}" _contents) + file(WRITE "${AWS_CONFIG}" "include(CMakeFindDependencyMacro)\nfind_dependency(aws-cpp-sdk-core)\n${_contents}") +endforeach() + +file(REMOVE_RECURSE + "${CURRENT_PACKAGES_DIR}/debug/include" + "${CURRENT_PACKAGES_DIR}/debug/share" + "${CURRENT_PACKAGES_DIR}/lib/pkgconfig" + "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig" + "${CURRENT_PACKAGES_DIR}/nuget" + "${CURRENT_PACKAGES_DIR}/debug/nuget" +) + +if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") + file(GLOB LIB_FILES ${CURRENT_PACKAGES_DIR}/bin/*.lib) + if(LIB_FILES) + file(COPY ${LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/lib) + file(REMOVE ${LIB_FILES}) + endif() + file(GLOB DEBUG_LIB_FILES ${CURRENT_PACKAGES_DIR}/debug/bin/*.lib) + if(DEBUG_LIB_FILES) + file(COPY ${DEBUG_LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) + file(REMOVE ${DEBUG_LIB_FILES}) + endif() + + file(APPEND "${CURRENT_PACKAGES_DIR}/include/aws/core/SDKConfig.h" "#ifndef USE_IMPORT_EXPORT\n#define USE_IMPORT_EXPORT\n#endif") +endif() + +configure_file("${CURRENT_PORT_DIR}/usage" "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" @ONLY) + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-sdk-cpp/usage b/ports/aws-sdk-cpp/usage new file mode 100644 index 000000000000..37e1f617b2cb --- /dev/null +++ b/ports/aws-sdk-cpp/usage @@ -0,0 +1,12 @@ +The package @PORT@:@TARGET_TRIPLET@ provides CMake targets: + + When using AWSSDK, AWSSDK_ROOT_DIR must be defined by the user. + find_package(AWSSDK CONFIG COMPONENTS core dynamodb kinesis s3 REQUIRED) + target_include_directories(main PRIVATE ${AWSSDK_INCLUDE_DIRS}) + target_link_libraries(main PRIVATE ${AWSSDK_LIBRARIES}) + + OR + + find_package(aws-cpp-sdk-core REQUIRED) + target_include_directories(main PRIVATE aws-cpp-sdk-core) + target_link_libraries(main PRIVATE aws-cpp-sdk-core) diff --git a/ports/aws-sdk-cpp/vcpkg.in.json b/ports/aws-sdk-cpp/vcpkg.in.json new file mode 100644 index 000000000000..e71d1612b2ca --- /dev/null +++ b/ports/aws-sdk-cpp/vcpkg.in.json @@ -0,0 +1,37 @@ +{ + "name": "aws-sdk-cpp", + "version": "1.11.352", + "description": "AWS SDK for C++", + "homepage": "https://github.com/aws/aws-sdk-cpp", + "license": "Apache-2.0", + "supports": "!(windows & arm) & !uwp", + "dependencies": [ + "aws-crt-cpp", + { + "name": "curl", + "default-features": false, + "features": [ + "ssl" + ], + "platform": "!uwp & !windows" + }, + { + "name": "openssl", + "platform": "!uwp & !windows" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + }, + "zlib" + ], + "default-features": [ + "dynamodb", + "kinesis", + "s3" + ] +} diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json new file mode 100644 index 000000000000..4d3b5eadca1e --- /dev/null +++ b/ports/aws-sdk-cpp/vcpkg.json @@ -0,0 +1,1276 @@ +{ + "$note": "Automatically generated by generateFeatures.ps1", + "name": "aws-sdk-cpp", + "version": "1.11.352", + "description": "AWS SDK for C++", + "homepage": "https://github.com/aws/aws-sdk-cpp", + "license": "Apache-2.0", + "supports": "!(windows & arm) & !uwp", + "dependencies": [ + "aws-crt-cpp", + { + "name": "curl", + "default-features": false, + "features": [ + "ssl" + ], + "platform": "!uwp & !windows" + }, + { + "name": "openssl", + "platform": "!uwp & !windows" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + }, + "zlib" + ], + "default-features": [ + "dynamodb", + "kinesis", + "s3" + ], + "features": { + "access-management": { + "description": "C++ SDK for the AWS access-management service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "cognito-identity", + "iam" + ] + } + ] + }, + "accessanalyzer": { + "description": "C++ SDK for the AWS accessanalyzer service" + }, + "account": { + "description": "C++ SDK for the AWS account service" + }, + "acm": { + "description": "C++ SDK for the AWS acm service" + }, + "acm-pca": { + "description": "C++ SDK for the AWS acm-pca service" + }, + "amp": { + "description": "C++ SDK for the AWS amp service" + }, + "amplify": { + "description": "C++ SDK for the AWS amplify service" + }, + "amplifybackend": { + "description": "C++ SDK for the AWS amplifybackend service" + }, + "amplifyuibuilder": { + "description": "C++ SDK for the AWS amplifyuibuilder service" + }, + "apigateway": { + "description": "C++ SDK for the AWS apigateway service" + }, + "apigatewaymanagementapi": { + "description": "C++ SDK for the AWS apigatewaymanagementapi service" + }, + "apigatewayv2": { + "description": "C++ SDK for the AWS apigatewayv2 service" + }, + "appconfig": { + "description": "C++ SDK for the AWS appconfig service" + }, + "appconfigdata": { + "description": "C++ SDK for the AWS appconfigdata service" + }, + "appfabric": { + "description": "C++ SDK for the AWS appfabric service" + }, + "appflow": { + "description": "C++ SDK for the AWS appflow service" + }, + "appintegrations": { + "description": "C++ SDK for the AWS appintegrations service" + }, + "application-autoscaling": { + "description": "C++ SDK for the AWS application-autoscaling service" + }, + "application-insights": { + "description": "C++ SDK for the AWS application-insights service" + }, + "application-signals": { + "description": "C++ SDK for the AWS application-signals service" + }, + "applicationcostprofiler": { + "description": "C++ SDK for the AWS applicationcostprofiler service" + }, + "appmesh": { + "description": "C++ SDK for the AWS appmesh service" + }, + "apprunner": { + "description": "C++ SDK for the AWS apprunner service" + }, + "appstream": { + "description": "C++ SDK for the AWS appstream service" + }, + "appsync": { + "description": "C++ SDK for the AWS appsync service" + }, + "apptest": { + "description": "C++ SDK for the AWS apptest service" + }, + "arc-zonal-shift": { + "description": "C++ SDK for the AWS arc-zonal-shift service" + }, + "artifact": { + "description": "C++ SDK for the AWS artifact service" + }, + "athena": { + "description": "C++ SDK for the AWS athena service" + }, + "auditmanager": { + "description": "C++ SDK for the AWS auditmanager service" + }, + "autoscaling": { + "description": "C++ SDK for the AWS autoscaling service" + }, + "autoscaling-plans": { + "description": "C++ SDK for the AWS autoscaling-plans service" + }, + "awsmigrationhub": { + "description": "C++ SDK for the AWS AWSMigrationHub service" + }, + "awstransfer": { + "description": "C++ SDK for the AWS awstransfer service" + }, + "b2bi": { + "description": "C++ SDK for the AWS b2bi service" + }, + "backup": { + "description": "C++ SDK for the AWS backup service" + }, + "backup-gateway": { + "description": "C++ SDK for the AWS backup-gateway service" + }, + "batch": { + "description": "C++ SDK for the AWS batch service" + }, + "bcm-data-exports": { + "description": "C++ SDK for the AWS bcm-data-exports service" + }, + "bedrock": { + "description": "C++ SDK for the AWS bedrock service" + }, + "bedrock-agent": { + "description": "C++ SDK for the AWS bedrock-agent service" + }, + "bedrock-agent-runtime": { + "description": "C++ SDK for the AWS bedrock-agent-runtime service" + }, + "bedrock-runtime": { + "description": "C++ SDK for the AWS bedrock-runtime service" + }, + "billingconductor": { + "description": "C++ SDK for the AWS billingconductor service" + }, + "braket": { + "description": "C++ SDK for the AWS braket service" + }, + "budgets": { + "description": "C++ SDK for the AWS budgets service" + }, + "ce": { + "description": "C++ SDK for the AWS ce service" + }, + "chatbot": { + "description": "C++ SDK for the AWS chatbot service" + }, + "chime": { + "description": "C++ SDK for the AWS chime service" + }, + "chime-sdk-identity": { + "description": "C++ SDK for the AWS chime-sdk-identity service" + }, + "chime-sdk-media-pipelines": { + "description": "C++ SDK for the AWS chime-sdk-media-pipelines service" + }, + "chime-sdk-meetings": { + "description": "C++ SDK for the AWS chime-sdk-meetings service" + }, + "chime-sdk-messaging": { + "description": "C++ SDK for the AWS chime-sdk-messaging service" + }, + "chime-sdk-voice": { + "description": "C++ SDK for the AWS chime-sdk-voice service" + }, + "cleanrooms": { + "description": "C++ SDK for the AWS cleanrooms service" + }, + "cleanroomsml": { + "description": "C++ SDK for the AWS cleanroomsml service" + }, + "cloud9": { + "description": "C++ SDK for the AWS cloud9 service" + }, + "cloudcontrol": { + "description": "C++ SDK for the AWS cloudcontrol service" + }, + "clouddirectory": { + "description": "C++ SDK for the AWS clouddirectory service" + }, + "cloudformation": { + "description": "C++ SDK for the AWS cloudformation service" + }, + "cloudfront": { + "description": "C++ SDK for the AWS cloudfront service" + }, + "cloudfront-keyvaluestore": { + "description": "C++ SDK for the AWS cloudfront-keyvaluestore service" + }, + "cloudhsm": { + "description": "C++ SDK for the AWS cloudhsm service" + }, + "cloudhsmv2": { + "description": "C++ SDK for the AWS cloudhsmv2 service" + }, + "cloudsearch": { + "description": "C++ SDK for the AWS cloudsearch service" + }, + "cloudsearchdomain": { + "description": "C++ SDK for the AWS cloudsearchdomain service" + }, + "cloudtrail": { + "description": "C++ SDK for the AWS cloudtrail service" + }, + "cloudtrail-data": { + "description": "C++ SDK for the AWS cloudtrail-data service" + }, + "codeartifact": { + "description": "C++ SDK for the AWS codeartifact service" + }, + "codebuild": { + "description": "C++ SDK for the AWS codebuild service" + }, + "codecatalyst": { + "description": "C++ SDK for the AWS codecatalyst service" + }, + "codecommit": { + "description": "C++ SDK for the AWS codecommit service" + }, + "codeconnections": { + "description": "C++ SDK for the AWS codeconnections service" + }, + "codedeploy": { + "description": "C++ SDK for the AWS codedeploy service" + }, + "codeguru-reviewer": { + "description": "C++ SDK for the AWS codeguru-reviewer service" + }, + "codeguru-security": { + "description": "C++ SDK for the AWS codeguru-security service" + }, + "codeguruprofiler": { + "description": "C++ SDK for the AWS codeguruprofiler service" + }, + "codepipeline": { + "description": "C++ SDK for the AWS codepipeline service" + }, + "codestar": { + "description": "C++ SDK for the AWS codestar service" + }, + "codestar-connections": { + "description": "C++ SDK for the AWS codestar-connections service" + }, + "codestar-notifications": { + "description": "C++ SDK for the AWS codestar-notifications service" + }, + "cognito-identity": { + "description": "C++ SDK for the AWS cognito-identity service" + }, + "cognito-idp": { + "description": "C++ SDK for the AWS cognito-idp service" + }, + "cognito-sync": { + "description": "C++ SDK for the AWS cognito-sync service" + }, + "comprehend": { + "description": "C++ SDK for the AWS comprehend service" + }, + "comprehendmedical": { + "description": "C++ SDK for the AWS comprehendmedical service" + }, + "compute-optimizer": { + "description": "C++ SDK for the AWS compute-optimizer service" + }, + "config": { + "description": "C++ SDK for the AWS config service" + }, + "connect": { + "description": "C++ SDK for the AWS connect service" + }, + "connect-contact-lens": { + "description": "C++ SDK for the AWS connect-contact-lens service" + }, + "connectcampaigns": { + "description": "C++ SDK for the AWS connectcampaigns service" + }, + "connectcases": { + "description": "C++ SDK for the AWS connectcases service" + }, + "connectparticipant": { + "description": "C++ SDK for the AWS connectparticipant service" + }, + "controlcatalog": { + "description": "C++ SDK for the AWS controlcatalog service" + }, + "controltower": { + "description": "C++ SDK for the AWS controltower service" + }, + "cost-optimization-hub": { + "description": "C++ SDK for the AWS cost-optimization-hub service" + }, + "cur": { + "description": "C++ SDK for the AWS cur service" + }, + "customer-profiles": { + "description": "C++ SDK for the AWS customer-profiles service" + }, + "databrew": { + "description": "C++ SDK for the AWS databrew service" + }, + "dataexchange": { + "description": "C++ SDK for the AWS dataexchange service" + }, + "datapipeline": { + "description": "C++ SDK for the AWS datapipeline service" + }, + "datasync": { + "description": "C++ SDK for the AWS datasync service" + }, + "datazone": { + "description": "C++ SDK for the AWS datazone service" + }, + "dax": { + "description": "C++ SDK for the AWS dax service" + }, + "deadline": { + "description": "C++ SDK for the AWS deadline service" + }, + "detective": { + "description": "C++ SDK for the AWS detective service" + }, + "devicefarm": { + "description": "C++ SDK for the AWS devicefarm service" + }, + "devops-guru": { + "description": "C++ SDK for the AWS devops-guru service" + }, + "directconnect": { + "description": "C++ SDK for the AWS directconnect service" + }, + "discovery": { + "description": "C++ SDK for the AWS discovery service" + }, + "dlm": { + "description": "C++ SDK for the AWS dlm service" + }, + "dms": { + "description": "C++ SDK for the AWS dms service" + }, + "docdb": { + "description": "C++ SDK for the AWS docdb service" + }, + "docdb-elastic": { + "description": "C++ SDK for the AWS docdb-elastic service" + }, + "drs": { + "description": "C++ SDK for the AWS drs service" + }, + "ds": { + "description": "C++ SDK for the AWS ds service" + }, + "dynamodb": { + "description": "C++ SDK for the AWS dynamodb service" + }, + "dynamodbstreams": { + "description": "C++ SDK for the AWS dynamodbstreams service" + }, + "ebs": { + "description": "C++ SDK for the AWS ebs service" + }, + "ec2": { + "description": "C++ SDK for the AWS ec2 service" + }, + "ec2-instance-connect": { + "description": "C++ SDK for the AWS ec2-instance-connect service" + }, + "ecr": { + "description": "C++ SDK for the AWS ecr service" + }, + "ecr-public": { + "description": "C++ SDK for the AWS ecr-public service" + }, + "ecs": { + "description": "C++ SDK for the AWS ecs service" + }, + "eks": { + "description": "C++ SDK for the AWS eks service" + }, + "eks-auth": { + "description": "C++ SDK for the AWS eks-auth service" + }, + "elastic-inference": { + "description": "C++ SDK for the AWS elastic-inference service" + }, + "elasticache": { + "description": "C++ SDK for the AWS elasticache service" + }, + "elasticbeanstalk": { + "description": "C++ SDK for the AWS elasticbeanstalk service" + }, + "elasticfilesystem": { + "description": "C++ SDK for the AWS elasticfilesystem service" + }, + "elasticloadbalancing": { + "description": "C++ SDK for the AWS elasticloadbalancing service" + }, + "elasticloadbalancingv2": { + "description": "C++ SDK for the AWS elasticloadbalancingv2 service" + }, + "elasticmapreduce": { + "description": "C++ SDK for the AWS elasticmapreduce service" + }, + "elastictranscoder": { + "description": "C++ SDK for the AWS elastictranscoder service" + }, + "email": { + "description": "C++ SDK for the AWS email service" + }, + "emr-containers": { + "description": "C++ SDK for the AWS emr-containers service" + }, + "emr-serverless": { + "description": "C++ SDK for the AWS emr-serverless service" + }, + "entityresolution": { + "description": "C++ SDK for the AWS entityresolution service" + }, + "es": { + "description": "C++ SDK for the AWS es service" + }, + "eventbridge": { + "description": "C++ SDK for the AWS eventbridge service" + }, + "events": { + "description": "C++ SDK for the AWS events service" + }, + "evidently": { + "description": "C++ SDK for the AWS evidently service" + }, + "finspace": { + "description": "C++ SDK for the AWS finspace service" + }, + "finspace-data": { + "description": "C++ SDK for the AWS finspace-data service" + }, + "firehose": { + "description": "C++ SDK for the AWS firehose service" + }, + "fis": { + "description": "C++ SDK for the AWS fis service" + }, + "fms": { + "description": "C++ SDK for the AWS fms service" + }, + "forecast": { + "description": "C++ SDK for the AWS forecast service" + }, + "forecastquery": { + "description": "C++ SDK for the AWS forecastquery service" + }, + "frauddetector": { + "description": "C++ SDK for the AWS frauddetector service" + }, + "freetier": { + "description": "C++ SDK for the AWS freetier service" + }, + "fsx": { + "description": "C++ SDK for the AWS fsx service" + }, + "gamelift": { + "description": "C++ SDK for the AWS gamelift service" + }, + "glacier": { + "description": "C++ SDK for the AWS glacier service" + }, + "globalaccelerator": { + "description": "C++ SDK for the AWS globalaccelerator service" + }, + "glue": { + "description": "C++ SDK for the AWS glue service" + }, + "grafana": { + "description": "C++ SDK for the AWS grafana service" + }, + "greengrass": { + "description": "C++ SDK for the AWS greengrass service" + }, + "greengrassv2": { + "description": "C++ SDK for the AWS greengrassv2 service" + }, + "groundstation": { + "description": "C++ SDK for the AWS groundstation service" + }, + "guardduty": { + "description": "C++ SDK for the AWS guardduty service" + }, + "health": { + "description": "C++ SDK for the AWS health service" + }, + "healthlake": { + "description": "C++ SDK for the AWS healthlake service" + }, + "iam": { + "description": "C++ SDK for the AWS iam service" + }, + "identity-management": { + "description": "C++ SDK for the AWS identity-management service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "cognito-identity", + "sts" + ] + } + ] + }, + "identitystore": { + "description": "C++ SDK for the AWS identitystore service" + }, + "imagebuilder": { + "description": "C++ SDK for the AWS imagebuilder service" + }, + "importexport": { + "description": "C++ SDK for the AWS importexport service" + }, + "inspector": { + "description": "C++ SDK for the AWS inspector service" + }, + "inspector-scan": { + "description": "C++ SDK for the AWS inspector-scan service" + }, + "inspector2": { + "description": "C++ SDK for the AWS inspector2 service" + }, + "internetmonitor": { + "description": "C++ SDK for the AWS internetmonitor service" + }, + "iot": { + "description": "C++ SDK for the AWS iot service" + }, + "iot-data": { + "description": "C++ SDK for the AWS iot-data service" + }, + "iot-jobs-data": { + "description": "C++ SDK for the AWS iot-jobs-data service" + }, + "iot1click-devices": { + "description": "C++ SDK for the AWS iot1click-devices service" + }, + "iot1click-projects": { + "description": "C++ SDK for the AWS iot1click-projects service" + }, + "iotanalytics": { + "description": "C++ SDK for the AWS iotanalytics service" + }, + "iotdeviceadvisor": { + "description": "C++ SDK for the AWS iotdeviceadvisor service" + }, + "iotevents": { + "description": "C++ SDK for the AWS iotevents service" + }, + "iotevents-data": { + "description": "C++ SDK for the AWS iotevents-data service" + }, + "iotfleethub": { + "description": "C++ SDK for the AWS iotfleethub service" + }, + "iotfleetwise": { + "description": "C++ SDK for the AWS iotfleetwise service" + }, + "iotsecuretunneling": { + "description": "C++ SDK for the AWS iotsecuretunneling service" + }, + "iotsitewise": { + "description": "C++ SDK for the AWS iotsitewise service" + }, + "iotthingsgraph": { + "description": "C++ SDK for the AWS iotthingsgraph service" + }, + "iottwinmaker": { + "description": "C++ SDK for the AWS iottwinmaker service" + }, + "iotwireless": { + "description": "C++ SDK for the AWS iotwireless service" + }, + "ivs": { + "description": "C++ SDK for the AWS ivs service" + }, + "ivs-realtime": { + "description": "C++ SDK for the AWS ivs-realtime service" + }, + "ivschat": { + "description": "C++ SDK for the AWS ivschat service" + }, + "kafka": { + "description": "C++ SDK for the AWS kafka service" + }, + "kafkaconnect": { + "description": "C++ SDK for the AWS kafkaconnect service" + }, + "kendra": { + "description": "C++ SDK for the AWS kendra service" + }, + "kendra-ranking": { + "description": "C++ SDK for the AWS kendra-ranking service" + }, + "keyspaces": { + "description": "C++ SDK for the AWS keyspaces service" + }, + "kinesis": { + "description": "C++ SDK for the AWS kinesis service" + }, + "kinesis-video-archived-media": { + "description": "C++ SDK for the AWS kinesis-video-archived-media service" + }, + "kinesis-video-media": { + "description": "C++ SDK for the AWS kinesis-video-media service" + }, + "kinesis-video-signaling": { + "description": "C++ SDK for the AWS kinesis-video-signaling service" + }, + "kinesis-video-webrtc-storage": { + "description": "C++ SDK for the AWS kinesis-video-webrtc-storage service" + }, + "kinesisanalytics": { + "description": "C++ SDK for the AWS kinesisanalytics service" + }, + "kinesisanalyticsv2": { + "description": "C++ SDK for the AWS kinesisanalyticsv2 service" + }, + "kinesisvideo": { + "description": "C++ SDK for the AWS kinesisvideo service" + }, + "kms": { + "description": "C++ SDK for the AWS kms service" + }, + "lakeformation": { + "description": "C++ SDK for the AWS lakeformation service" + }, + "lambda": { + "description": "C++ SDK for the AWS lambda service" + }, + "launch-wizard": { + "description": "C++ SDK for the AWS launch-wizard service" + }, + "lex": { + "description": "C++ SDK for the AWS lex service" + }, + "lex-models": { + "description": "C++ SDK for the AWS lex-models service" + }, + "lexv2-models": { + "description": "C++ SDK for the AWS lexv2-models service" + }, + "lexv2-runtime": { + "description": "C++ SDK for the AWS lexv2-runtime service" + }, + "license-manager": { + "description": "C++ SDK for the AWS license-manager service" + }, + "license-manager-linux-subscriptions": { + "description": "C++ SDK for the AWS license-manager-linux-subscriptions service" + }, + "license-manager-user-subscriptions": { + "description": "C++ SDK for the AWS license-manager-user-subscriptions service" + }, + "lightsail": { + "description": "C++ SDK for the AWS lightsail service" + }, + "location": { + "description": "C++ SDK for the AWS location service" + }, + "logs": { + "description": "C++ SDK for the AWS logs service" + }, + "lookoutequipment": { + "description": "C++ SDK for the AWS lookoutequipment service" + }, + "lookoutmetrics": { + "description": "C++ SDK for the AWS lookoutmetrics service" + }, + "lookoutvision": { + "description": "C++ SDK for the AWS lookoutvision service" + }, + "m2": { + "description": "C++ SDK for the AWS m2 service" + }, + "machinelearning": { + "description": "C++ SDK for the AWS machinelearning service" + }, + "macie2": { + "description": "C++ SDK for the AWS macie2 service" + }, + "mailmanager": { + "description": "C++ SDK for the AWS mailmanager service" + }, + "managedblockchain": { + "description": "C++ SDK for the AWS managedblockchain service" + }, + "managedblockchain-query": { + "description": "C++ SDK for the AWS managedblockchain-query service" + }, + "marketplace-agreement": { + "description": "C++ SDK for the AWS marketplace-agreement service" + }, + "marketplace-catalog": { + "description": "C++ SDK for the AWS marketplace-catalog service" + }, + "marketplace-deployment": { + "description": "C++ SDK for the AWS marketplace-deployment service" + }, + "marketplace-entitlement": { + "description": "C++ SDK for the AWS marketplace-entitlement service" + }, + "marketplacecommerceanalytics": { + "description": "C++ SDK for the AWS marketplacecommerceanalytics service" + }, + "mediaconnect": { + "description": "C++ SDK for the AWS mediaconnect service" + }, + "mediaconvert": { + "description": "C++ SDK for the AWS mediaconvert service" + }, + "medialive": { + "description": "C++ SDK for the AWS medialive service" + }, + "mediapackage": { + "description": "C++ SDK for the AWS mediapackage service" + }, + "mediapackage-vod": { + "description": "C++ SDK for the AWS mediapackage-vod service" + }, + "mediapackagev2": { + "description": "C++ SDK for the AWS mediapackagev2 service" + }, + "mediastore": { + "description": "C++ SDK for the AWS mediastore service" + }, + "mediastore-data": { + "description": "C++ SDK for the AWS mediastore-data service" + }, + "mediatailor": { + "description": "C++ SDK for the AWS mediatailor service" + }, + "medical-imaging": { + "description": "C++ SDK for the AWS medical-imaging service" + }, + "memorydb": { + "description": "C++ SDK for the AWS memorydb service" + }, + "meteringmarketplace": { + "description": "C++ SDK for the AWS meteringmarketplace service" + }, + "mgn": { + "description": "C++ SDK for the AWS mgn service" + }, + "migration-hub-refactor-spaces": { + "description": "C++ SDK for the AWS migration-hub-refactor-spaces service" + }, + "migrationhub-config": { + "description": "C++ SDK for the AWS migrationhub-config service" + }, + "migrationhuborchestrator": { + "description": "C++ SDK for the AWS migrationhuborchestrator service" + }, + "migrationhubstrategy": { + "description": "C++ SDK for the AWS migrationhubstrategy service" + }, + "mobile": { + "description": "C++ SDK for the AWS mobile service" + }, + "monitoring": { + "description": "C++ SDK for the AWS monitoring service" + }, + "mq": { + "description": "C++ SDK for the AWS mq service" + }, + "mturk-requester": { + "description": "C++ SDK for the AWS mturk-requester service" + }, + "mwaa": { + "description": "C++ SDK for the AWS mwaa service" + }, + "neptune": { + "description": "C++ SDK for the AWS neptune service" + }, + "neptune-graph": { + "description": "C++ SDK for the AWS neptune-graph service" + }, + "neptunedata": { + "description": "C++ SDK for the AWS neptunedata service" + }, + "network-firewall": { + "description": "C++ SDK for the AWS network-firewall service" + }, + "networkmanager": { + "description": "C++ SDK for the AWS networkmanager service" + }, + "networkmonitor": { + "description": "C++ SDK for the AWS networkmonitor service" + }, + "nimble": { + "description": "C++ SDK for the AWS nimble service" + }, + "oam": { + "description": "C++ SDK for the AWS oam service" + }, + "omics": { + "description": "C++ SDK for the AWS omics service" + }, + "opensearch": { + "description": "C++ SDK for the AWS opensearch service" + }, + "opensearchserverless": { + "description": "C++ SDK for the AWS opensearchserverless service" + }, + "opsworks": { + "description": "C++ SDK for the AWS opsworks service" + }, + "opsworkscm": { + "description": "C++ SDK for the AWS opsworkscm service" + }, + "organizations": { + "description": "C++ SDK for the AWS organizations service" + }, + "osis": { + "description": "C++ SDK for the AWS osis service" + }, + "outposts": { + "description": "C++ SDK for the AWS outposts service" + }, + "panorama": { + "description": "C++ SDK for the AWS panorama service" + }, + "payment-cryptography": { + "description": "C++ SDK for the AWS payment-cryptography service" + }, + "payment-cryptography-data": { + "description": "C++ SDK for the AWS payment-cryptography-data service" + }, + "pca-connector-ad": { + "description": "C++ SDK for the AWS pca-connector-ad service" + }, + "pca-connector-scep": { + "description": "C++ SDK for the AWS pca-connector-scep service" + }, + "personalize": { + "description": "C++ SDK for the AWS personalize service" + }, + "personalize-events": { + "description": "C++ SDK for the AWS personalize-events service" + }, + "personalize-runtime": { + "description": "C++ SDK for the AWS personalize-runtime service" + }, + "pi": { + "description": "C++ SDK for the AWS pi service" + }, + "pinpoint": { + "description": "C++ SDK for the AWS pinpoint service" + }, + "pinpoint-email": { + "description": "C++ SDK for the AWS pinpoint-email service" + }, + "pinpoint-sms-voice-v2": { + "description": "C++ SDK for the AWS pinpoint-sms-voice-v2 service" + }, + "pipes": { + "description": "C++ SDK for the AWS pipes service" + }, + "polly": { + "description": "C++ SDK for the AWS polly service" + }, + "pricing": { + "description": "C++ SDK for the AWS pricing service" + }, + "privatenetworks": { + "description": "C++ SDK for the AWS privatenetworks service" + }, + "proton": { + "description": "C++ SDK for the AWS proton service" + }, + "qbusiness": { + "description": "C++ SDK for the AWS qbusiness service" + }, + "qconnect": { + "description": "C++ SDK for the AWS qconnect service" + }, + "qldb": { + "description": "C++ SDK for the AWS qldb service" + }, + "qldb-session": { + "description": "C++ SDK for the AWS qldb-session service" + }, + "queues": { + "description": "C++ SDK for the AWS queues service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "sqs" + ] + } + ] + }, + "quicksight": { + "description": "C++ SDK for the AWS quicksight service" + }, + "ram": { + "description": "C++ SDK for the AWS ram service" + }, + "rbin": { + "description": "C++ SDK for the AWS rbin service" + }, + "rds": { + "description": "C++ SDK for the AWS rds service" + }, + "rds-data": { + "description": "C++ SDK for the AWS rds-data service" + }, + "redshift": { + "description": "C++ SDK for the AWS redshift service" + }, + "redshift-data": { + "description": "C++ SDK for the AWS redshift-data service" + }, + "redshift-serverless": { + "description": "C++ SDK for the AWS redshift-serverless service" + }, + "rekognition": { + "description": "C++ SDK for the AWS rekognition service" + }, + "repostspace": { + "description": "C++ SDK for the AWS repostspace service" + }, + "resiliencehub": { + "description": "C++ SDK for the AWS resiliencehub service" + }, + "resource-explorer-2": { + "description": "C++ SDK for the AWS resource-explorer-2 service" + }, + "resource-groups": { + "description": "C++ SDK for the AWS resource-groups service" + }, + "resourcegroupstaggingapi": { + "description": "C++ SDK for the AWS resourcegroupstaggingapi service" + }, + "robomaker": { + "description": "C++ SDK for the AWS robomaker service" + }, + "rolesanywhere": { + "description": "C++ SDK for the AWS rolesanywhere service" + }, + "route53": { + "description": "C++ SDK for the AWS route53 service" + }, + "route53-recovery-cluster": { + "description": "C++ SDK for the AWS route53-recovery-cluster service" + }, + "route53-recovery-control-config": { + "description": "C++ SDK for the AWS route53-recovery-control-config service" + }, + "route53-recovery-readiness": { + "description": "C++ SDK for the AWS route53-recovery-readiness service" + }, + "route53domains": { + "description": "C++ SDK for the AWS route53domains service" + }, + "route53profiles": { + "description": "C++ SDK for the AWS route53profiles service" + }, + "route53resolver": { + "description": "C++ SDK for the AWS route53resolver service" + }, + "rum": { + "description": "C++ SDK for the AWS rum service" + }, + "s3": { + "description": "C++ SDK for the AWS s3 service" + }, + "s3-crt": { + "description": "C++ SDK for the AWS s3-crt service" + }, + "s3-encryption": { + "description": "C++ SDK for the AWS s3-encryption service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "kms", + "s3" + ] + } + ] + }, + "s3control": { + "description": "C++ SDK for the AWS s3control service" + }, + "s3outposts": { + "description": "C++ SDK for the AWS s3outposts service" + }, + "sagemaker": { + "description": "C++ SDK for the AWS sagemaker service" + }, + "sagemaker-a2i-runtime": { + "description": "C++ SDK for the AWS sagemaker-a2i-runtime service" + }, + "sagemaker-edge": { + "description": "C++ SDK for the AWS sagemaker-edge service" + }, + "sagemaker-featurestore-runtime": { + "description": "C++ SDK for the AWS sagemaker-featurestore-runtime service" + }, + "sagemaker-geospatial": { + "description": "C++ SDK for the AWS sagemaker-geospatial service" + }, + "sagemaker-metrics": { + "description": "C++ SDK for the AWS sagemaker-metrics service" + }, + "sagemaker-runtime": { + "description": "C++ SDK for the AWS sagemaker-runtime service" + }, + "savingsplans": { + "description": "C++ SDK for the AWS savingsplans service" + }, + "scheduler": { + "description": "C++ SDK for the AWS scheduler service" + }, + "schemas": { + "description": "C++ SDK for the AWS schemas service" + }, + "sdb": { + "description": "C++ SDK for the AWS sdb service" + }, + "secretsmanager": { + "description": "C++ SDK for the AWS secretsmanager service" + }, + "securityhub": { + "description": "C++ SDK for the AWS securityhub service" + }, + "securitylake": { + "description": "C++ SDK for the AWS securitylake service" + }, + "serverlessrepo": { + "description": "C++ SDK for the AWS serverlessrepo service" + }, + "service-quotas": { + "description": "C++ SDK for the AWS service-quotas service" + }, + "servicecatalog": { + "description": "C++ SDK for the AWS servicecatalog service" + }, + "servicecatalog-appregistry": { + "description": "C++ SDK for the AWS servicecatalog-appregistry service" + }, + "servicediscovery": { + "description": "C++ SDK for the AWS servicediscovery service" + }, + "sesv2": { + "description": "C++ SDK for the AWS sesv2 service" + }, + "shield": { + "description": "C++ SDK for the AWS shield service" + }, + "signer": { + "description": "C++ SDK for the AWS signer service" + }, + "simspaceweaver": { + "description": "C++ SDK for the AWS simspaceweaver service" + }, + "sms": { + "description": "C++ SDK for the AWS sms service" + }, + "sms-voice": { + "description": "C++ SDK for the AWS sms-voice service" + }, + "snow-device-management": { + "description": "C++ SDK for the AWS snow-device-management service" + }, + "snowball": { + "description": "C++ SDK for the AWS snowball service" + }, + "sns": { + "description": "C++ SDK for the AWS sns service" + }, + "sqs": { + "description": "C++ SDK for the AWS sqs service" + }, + "ssm": { + "description": "C++ SDK for the AWS ssm service" + }, + "ssm-contacts": { + "description": "C++ SDK for the AWS ssm-contacts service" + }, + "ssm-incidents": { + "description": "C++ SDK for the AWS ssm-incidents service" + }, + "ssm-sap": { + "description": "C++ SDK for the AWS ssm-sap service" + }, + "sso": { + "description": "C++ SDK for the AWS sso service" + }, + "sso-admin": { + "description": "C++ SDK for the AWS sso-admin service" + }, + "sso-oidc": { + "description": "C++ SDK for the AWS sso-oidc service" + }, + "states": { + "description": "C++ SDK for the AWS states service" + }, + "storagegateway": { + "description": "C++ SDK for the AWS storagegateway service" + }, + "sts": { + "description": "C++ SDK for the AWS sts service" + }, + "supplychain": { + "description": "C++ SDK for the AWS supplychain service" + }, + "support": { + "description": "C++ SDK for the AWS support service" + }, + "support-app": { + "description": "C++ SDK for the AWS support-app service" + }, + "swf": { + "description": "C++ SDK for the AWS swf service" + }, + "synthetics": { + "description": "C++ SDK for the AWS synthetics service" + }, + "taxsettings": { + "description": "C++ SDK for the AWS taxsettings service" + }, + "text-to-speech": { + "description": "C++ SDK for the AWS text-to-speech service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "polly" + ] + } + ] + }, + "textract": { + "description": "C++ SDK for the AWS textract service" + }, + "timestream-influxdb": { + "description": "C++ SDK for the AWS timestream-influxdb service" + }, + "timestream-query": { + "description": "C++ SDK for the AWS timestream-query service" + }, + "timestream-write": { + "description": "C++ SDK for the AWS timestream-write service" + }, + "tnb": { + "description": "C++ SDK for the AWS tnb service" + }, + "transcribe": { + "description": "C++ SDK for the AWS transcribe service" + }, + "transcribestreaming": { + "description": "C++ SDK for the AWS transcribestreaming service" + }, + "transfer": { + "description": "C++ SDK for the AWS transfer service", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "default-features": false, + "features": [ + "s3" + ] + } + ] + }, + "translate": { + "description": "C++ SDK for the AWS translate service" + }, + "trustedadvisor": { + "description": "C++ SDK for the AWS trustedadvisor service" + }, + "verifiedpermissions": { + "description": "C++ SDK for the AWS verifiedpermissions service" + }, + "voice-id": { + "description": "C++ SDK for the AWS voice-id service" + }, + "vpc-lattice": { + "description": "C++ SDK for the AWS vpc-lattice service" + }, + "waf": { + "description": "C++ SDK for the AWS waf service" + }, + "waf-regional": { + "description": "C++ SDK for the AWS waf-regional service" + }, + "wafv2": { + "description": "C++ SDK for the AWS wafv2 service" + }, + "wellarchitected": { + "description": "C++ SDK for the AWS wellarchitected service" + }, + "wisdom": { + "description": "C++ SDK for the AWS wisdom service" + }, + "workdocs": { + "description": "C++ SDK for the AWS workdocs service" + }, + "worklink": { + "description": "C++ SDK for the AWS worklink service" + }, + "workmail": { + "description": "C++ SDK for the AWS workmail service" + }, + "workmailmessageflow": { + "description": "C++ SDK for the AWS workmailmessageflow service" + }, + "workspaces": { + "description": "C++ SDK for the AWS workspaces service" + }, + "workspaces-thin-client": { + "description": "C++ SDK for the AWS workspaces-thin-client service" + }, + "workspaces-web": { + "description": "C++ SDK for the AWS workspaces-web service" + }, + "xray": { + "description": "C++ SDK for the AWS xray service" + } + } +} diff --git a/ports/azure-storage-common-cpp/portfile.cmake b/ports/azure-storage-common-cpp/portfile.cmake deleted file mode 100644 index 482ed9687f13..000000000000 --- a/ports/azure-storage-common-cpp/portfile.cmake +++ /dev/null @@ -1,32 +0,0 @@ -# NOTE: All changes made to this file will get overwritten by the next port release. -# Please contribute your changes to https://github.com/Azure/azure-sdk-for-cpp. - -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO Azure/azure-sdk-for-cpp - REF azure-storage-common_12.5.0 - SHA512 46c9cc4ebec54f289ba7270356da4e89e5f8e890004c7a232200b87ca33357064c2f46670a1090fe41ca6962cdbc76d2e3520bb600438cbc0f21f15cf7816f04 -) - -if(EXISTS "${SOURCE_PATH}/sdk/storage/azure-storage-common") - file(REMOVE_RECURSE "${SOURCE_PATH}/sdk/storage/_") - file(REMOVE_RECURSE "${SOURCE_PATH}/sdk/_") - file(REMOVE_RECURSE "${SOURCE_PATH}/_") - - file(RENAME "${SOURCE_PATH}/sdk/storage/azure-storage-common" "${SOURCE_PATH}/sdk/storage/_") - file(RENAME "${SOURCE_PATH}/sdk/storage" "${SOURCE_PATH}/sdk/_") - file(RENAME "${SOURCE_PATH}/sdk" "${SOURCE_PATH}/_") -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}/_/_/_" - OPTIONS - -DWARNINGS_AS_ERRORS=OFF - -DBUILD_TESTING=OFF -) - -vcpkg_cmake_install() -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -vcpkg_cmake_config_fixup() -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -vcpkg_copy_pdbs() diff --git a/ports/azure-storage-common-cpp/vcpkg.json b/ports/azure-storage-common-cpp/vcpkg.json deleted file mode 100644 index 8dd26d908204..000000000000 --- a/ports/azure-storage-common-cpp/vcpkg.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$comment": "NOTE: All changes made to this file will get overwritten by the next port release. Please contribute your changes to https://github.com/Azure/azure-sdk-for-cpp.", - "name": "azure-storage-common-cpp", - "version-semver": "12.5.0", - "port-version": 1, - "description": [ - "Microsoft Azure Common Storage SDK for C++", - "This library provides common Azure Storage-related abstractions for Azure SDK." - ], - "homepage": "https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/storage/azure-storage-common", - "license": "MIT", - "dependencies": [ - { - "name": "azure-core-cpp", - "default-features": false, - "version>=": "1.10.3" - }, - { - "name": "libxml2", - "default-features": false, - "platform": "!windows" - }, - { - "name": "openssl", - "platform": "!windows" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/google-cloud-cpp/portfile.cmake b/ports/google-cloud-cpp/portfile.cmake deleted file mode 100644 index 4f9674ea9e0a..000000000000 --- a/ports/google-cloud-cpp/portfile.cmake +++ /dev/null @@ -1,86 +0,0 @@ -vcpkg_check_linkage(ONLY_STATIC_LIBRARY) - -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO googleapis/google-cloud-cpp - REF "v${VERSION}" - SHA512 a392da19ac353409ecbf30e390803b2e34670552fd54466a08ee554a77c893d447289b30d5841c7f79b2a23244a269d8ecac8f7fbd8a34dda45ce2d8b1d46817 - HEAD_REF main - PATCHES - support_absl_cxx17.patch - remove_gmock.patch -) - -if ("grpc-common" IN_LIST FEATURES) - vcpkg_add_to_path(PREPEND "${CURRENT_HOST_INSTALLED_DIR}/tools/grpc") -endif () - -set(GOOGLE_CLOUD_CPP_ENABLE "${FEATURES}") -list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "core") -# This feature does not exist, but allows us to simplify the vcpkg.json -# file. -list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "grpc-common") -list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "rest-common") -list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "googleapis") -# google-cloud-cpp uses dialogflow_cx and dialogflow_es. Underscores -# are invalid in `vcpkg` features, we use dashes (`-`) as a separator -# for the `vcpkg` feature name, and convert it here to something that -# `google-cloud-cpp` would like. -if ("dialogflow-cx" IN_LIST FEATURES) - list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "dialogflow-cx") - list(APPEND GOOGLE_CLOUD_CPP_ENABLE "dialogflow_cx") -endif () -if ("dialogflow-es" IN_LIST FEATURES) - list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "dialogflow-es") - list(APPEND GOOGLE_CLOUD_CPP_ENABLE "dialogflow_es") -endif () - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - "-DGOOGLE_CLOUD_CPP_ENABLE=${GOOGLE_CLOUD_CPP_ENABLE}" - -DGOOGLE_CLOUD_CPP_ENABLE_MACOS_OPENSSL_CHECK=OFF - -DGOOGLE_CLOUD_CPP_ENABLE_WERROR=OFF - -DGOOGLE_CLOUD_CPP_ENABLE_CCACHE=OFF - -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF - -DBUILD_TESTING=OFF - # This is needed by the `experimental-storage-grpc` feature until vcpkg - # gets Protobuf >= 4.23.0. It has no effect for other features, so - # it is simpler to just always turn it on. - -DGOOGLE_CLOUD_CPP_ENABLE_CTYPE_CORD_WORKAROUND=ON -) - -vcpkg_cmake_install() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -foreach(feature IN LISTS FEATURES) - set(config_path "lib/cmake/google_cloud_cpp_${feature}") - # Most features get their own package in `google-cloud-cpp`. - # The exceptions are captured by this `if()` command, basically - # things like `core` and `experimental-storage-grpc` are skipped. - if(NOT IS_DIRECTORY "${CURRENT_PACKAGES_DIR}/${config_path}") - continue() - endif() - vcpkg_cmake_config_fixup(PACKAGE_NAME "google_cloud_cpp_${feature}" - CONFIG_PATH "${config_path}" - DO_NOT_DELETE_PARENT_CONFIG_PATH) -endforeach() -# These packages are automatically installed depending on what features are -# enabled. -foreach(suffix common compute_protos googleapis grpc_utils iam_v2 logging_type rest_internal rest_protobuf_internal dialogflow_cx dialogflow_es) - set(config_path "lib/cmake/google_cloud_cpp_${suffix}") - if(NOT IS_DIRECTORY "${CURRENT_PACKAGES_DIR}/${config_path}") - continue() - endif() - vcpkg_cmake_config_fixup(PACKAGE_NAME "google_cloud_cpp_${suffix}" - CONFIG_PATH "${config_path}" - DO_NOT_DELETE_PARENT_CONFIG_PATH) -endforeach() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/lib/cmake" - "${CURRENT_PACKAGES_DIR}/debug/lib/cmake" - "${CURRENT_PACKAGES_DIR}/debug/share") -file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) - -vcpkg_copy_pdbs() diff --git a/ports/google-cloud-cpp/remove_gmock.patch b/ports/google-cloud-cpp/remove_gmock.patch deleted file mode 100644 index 304640345752..000000000000 --- a/ports/google-cloud-cpp/remove_gmock.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/google/cloud/google_cloud_cpp_common.cmake b/google/cloud/google_cloud_cpp_common.cmake -index 1a14c77..4e8058e 100644 ---- a/google/cloud/google_cloud_cpp_common.cmake -+++ b/google/cloud/google_cloud_cpp_common.cmake -@@ -271,9 +271,6 @@ set(google_cloud_cpp_mocks_hdrs - mocks/mock_stream_range.h) - export_list_to_bazel("google_cloud_cpp_mocks.bzl" "google_cloud_cpp_mocks_hdrs" - YEAR "2022") --target_link_libraries( -- google_cloud_cpp_mocks INTERFACE google-cloud-cpp::common GTest::gmock_main -- GTest::gmock GTest::gtest) - set_target_properties(google_cloud_cpp_mocks PROPERTIES EXPORT_NAME - google-cloud-cpp::mocks) - target_include_directories( -diff --git a/google/cloud/storage/google_cloud_cpp_storage_grpc.cmake b/google/cloud/storage/google_cloud_cpp_storage_grpc.cmake -index a12bb20..eb3182f 100644 ---- a/google/cloud/storage/google_cloud_cpp_storage_grpc.cmake -+++ b/google/cloud/storage/google_cloud_cpp_storage_grpc.cmake -@@ -219,9 +219,6 @@ set(google_cloud_cpp_storage_grpc_mocks_hdrs - mocks/mock_async_writer_connection.h) - export_list_to_bazel("google_cloud_cpp_storage_grpc_mocks.bzl" - "google_cloud_cpp_storage_grpc_mocks_hdrs" YEAR "2023") --target_link_libraries( -- google_cloud_cpp_storage_grpc_mocks -- INTERFACE google-cloud-cpp::experimental-storage_grpc GTest::gmock) - set_target_properties( - google_cloud_cpp_storage_grpc_mocks - PROPERTIES EXPORT_NAME "google-cloud-cpp::experimental-storage_grpc_mocks") diff --git a/ports/google-cloud-cpp/support_absl_cxx17.patch b/ports/google-cloud-cpp/support_absl_cxx17.patch deleted file mode 100644 index 582b88a52d40..000000000000 --- a/ports/google-cloud-cpp/support_absl_cxx17.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 3fb0564..b4a251b 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -29,6 +29,14 @@ if (NOT "${PROJECT_VERSION_PRE_RELEASE}" STREQUAL "") - set(PROJECT_VERSION "${PROJECT_VERSION}-${PROJECT_VERSION_PRE_RELEASE}") - endif () - -+find_package(absl CONFIG REQUIRED) -+ -+# Use CMAKE_CXX_STANDARD=17 if ABSL_USE_CXX17 is set -+if (ABSL_USE_CXX17) -+ message(STATUS "Found absl uses CXX17, enable CXX17 feature.") -+ set(CMAKE_CXX_STANDARD 17) -+endif () -+ - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.3) - message( diff --git a/ports/google-cloud-cpp/vcpkg.json b/ports/google-cloud-cpp/vcpkg.json deleted file mode 100644 index 0c5ac9338bd1..000000000000 --- a/ports/google-cloud-cpp/vcpkg.json +++ /dev/null @@ -1,1509 +0,0 @@ -{ - "name": "google-cloud-cpp", - "version": "2.19.0", - "description": "C++ Client Libraries for Google Cloud Platform APIs.", - "homepage": "https://github.com/googleapis/google-cloud-cpp", - "license": "Apache-2.0", - "supports": "!uwp", - "dependencies": [ - "abseil", - "openssl", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "bigquery", - "bigtable", - "iam", - "pubsub", - "spanner", - "storage" - ], - "features": { - "accessapproval": { - "description": "Access Approval API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "accesscontextmanager": { - "description": "Access Context Manager API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "advisorynotifications": { - "description": "Advisory Notifications API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "alloydb": { - "description": "Alloy DB API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "apigateway": { - "description": "API Gateway API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "apigeeconnect": { - "description": "Apigee Connect API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "apikeys": { - "description": "API Keys API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "appengine": { - "description": "App Engine Admin API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "artifactregistry": { - "description": "Artifact Registry API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "asset": { - "description": "Cloud Asset API C++ Client Library", - "supports": "!windows", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "accesscontextmanager", - "grpc-common", - "osconfig" - ] - } - ] - }, - "assuredworkloads": { - "description": "Assured Workloads API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "automl": { - "description": "Cloud AutoML API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "baremetalsolution": { - "description": "Bare Metal Solution API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "batch": { - "description": "Batch API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "beyondcorp": { - "description": "BeyondCorp API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "bigquery": { - "description": "The Google Cloud BigQuery C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "bigtable": { - "description": "The Google Cloud Bigtable C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "billing": { - "description": "Cloud Billing Budget API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "binaryauthorization": { - "description": "Binary Authorization API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grafeas", - "grpc-common" - ] - } - ] - }, - "certificatemanager": { - "description": "Certificate Manager API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "channel": { - "description": "Cloud Channel API C++ Client Library", - "supports": "!windows", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "cloudbuild": { - "description": "Cloud Build API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "commerce": { - "description": "Cloud Commerce C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "composer": { - "description": "Cloud Composer C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "compute": { - "description": "Compute Engine C++ Client Library", - "supports": "!windows", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common", - "rest-common" - ] - } - ] - }, - "confidentialcomputing": { - "description": "Confidential Computing API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "config": { - "description": "Infrastructure Manager API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "connectors": { - "description": "Connectors API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "contactcenterinsights": { - "description": "Contact Center AI Insights API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "container": { - "description": "Kubernetes Engine API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "containeranalysis": { - "description": "Container Analysis API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grafeas", - "grpc-common" - ] - } - ] - }, - "datacatalog": { - "description": "Google Cloud Data Catalog API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "datafusion": { - "description": "Cloud Data Fusion API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "datamigration": { - "description": "Database Migration API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dataplex": { - "description": "Cloud Dataplex API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dataproc": { - "description": "Cloud Dataproc API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "datastore": { - "description": "Cloud Datastore API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "datastream": { - "description": "Datastream API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "deploy": { - "description": "Google Cloud Deploy API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dialogflow-cx": { - "description": "Cloud Dialogflow CX API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dialogflow-es": { - "description": "Cloud Dialogflow ES API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "discoveryengine": { - "description": "Discovery Engine API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dlp": { - "description": "Cloud Data Loss Prevention (DLP) API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "documentai": { - "description": "Cloud Document AI API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "edgecontainer": { - "description": "Distributed Cloud Edge Container API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "edgenetwork": { - "description": "Distributed Cloud Edge Network API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "eventarc": { - "description": "Eventarc API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "experimental-storage-grpc": { - "description": "The GCS+gRPC plugin", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common", - "storage" - ] - } - ] - }, - "filestore": { - "description": "Cloud Filestore API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "functions": { - "description": "Cloud Functions API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "gkehub": { - "description": "GKE Hub C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "gkemulticloud": { - "description": "Anthos Multi-Cloud C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "grafeas": { - "description": "Protocol buffers implementing the 'Grafeas API' (metadata about software artifacts)", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "grpc-common": { - "description": "Dependencies used by all gRPC-based libraries", - "dependencies": [ - "grpc", - { - "name": "grpc", - "host": true - }, - "protobuf", - { - "name": "protobuf", - "host": true - } - ] - }, - "iam": { - "description": "The Google Cloud IAM C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "iap": { - "description": "Cloud Identity-Aware Proxy API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "ids": { - "description": "Cloud IDS API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "kms": { - "description": "Cloud Key Management Service (KMS) API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "language": { - "description": "Cloud Natural Language API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "logging": { - "description": "Google Cloud Logging C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "managedidentities": { - "description": "Managed Service for Microsoft Active Directory API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "memcache": { - "description": "Cloud Memorystore for Memcached API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "metastore": { - "description": "Dataproc Metastore API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "migrationcenter": { - "description": "Migration Center API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "monitoring": { - "description": "Cloud Monitoring API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "netapp": { - "description": "NetApp API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "networkconnectivity": { - "description": "Network Connectivity API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "networkmanagement": { - "description": "Network Management API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "networksecurity": { - "description": "Secure Web Proxy API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "notebooks": { - "description": "Notebooks API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "oauth2": { - "description": "OAuth2 Access Token Generation Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "rest-common" - ] - } - ] - }, - "opentelemetry": { - "description": "OpenTelemetry C++ GCP Exporter Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "rest-common", - "trace" - ] - }, - "opentelemetry-cpp" - ] - }, - "optimization": { - "description": "Cloud Optimization API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "orgpolicy": { - "description": "Organization Policy API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "osconfig": { - "description": "Cloud OS Config API C++ Client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "oslogin": { - "description": "Cloud OS Login API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "policysimulator": { - "description": "Policy Simulator API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "policytroubleshooter": { - "description": "Policy Troubleshooter API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "privateca": { - "description": "Certificate Authority API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "profiler": { - "description": "Cloud Profiler API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "pubsub": { - "description": "The Google Cloud Bigtable C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "rapidmigrationassessment": { - "description": "Rapid Migration Assessment C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "recommender": { - "description": "Recommender C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "redis": { - "description": "Google Cloud Memorystore for Redis API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "resourcemanager": { - "description": "Cloud Resource Manager API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "resourcesettings": { - "description": "Resource Settings API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "rest-common": { - "description": "Dependencies used by all REST-based libraries", - "dependencies": [ - { - "name": "curl", - "features": [ - "ssl" - ] - }, - "nlohmann-json" - ] - }, - "retail": { - "description": "Retail API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "run": { - "description": "Cloud Run Admin API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "scheduler": { - "description": "Cloud Scheduler API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "secretmanager": { - "description": "The Google Cloud Secret Manager C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "securesourcemanager": { - "description": "Secure Source Manager API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "securitycenter": { - "description": "Security Command Center API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "servicecontrol": { - "description": "Service Control API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "servicedirectory": { - "description": "Service Directory API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "servicemanagement": { - "description": "Service Management API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "serviceusage": { - "description": "Service Usage API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "shell": { - "description": "Cloud Shell API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "spanner": { - "description": "The Google Cloud Spanner C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "speech": { - "description": "The Google Cloud Speech-to-Text C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "sql": { - "description": "The Google Cloud SQL Admin C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "rest-common" - ] - } - ] - }, - "storage": { - "description": "The Google Cloud Storage C++ client library", - "dependencies": [ - "crc32c", - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "rest-common" - ] - } - ] - }, - "storageinsights": { - "description": "Storage Insights API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "storagetransfer": { - "description": "Storage Transfer API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "support": { - "description": "Cloud Support API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "talent": { - "description": "Cloud Talent Solution API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "tasks": { - "description": "The Google Cloud Tasks C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "telcoautomation": { - "description": "Cloud Telco Automation API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "texttospeech": { - "description": "Cloud Text-to-Speech API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "tpu": { - "description": "Cloud TPU API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "trace": { - "description": "Stackdriver Trace API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "translate": { - "description": "Cloud Translation API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "video": { - "description": "Video Services C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "videointelligence": { - "description": "Cloud Video Intelligence API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "vision": { - "description": "Cloud Vision API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "vmmigration": { - "description": "VM Migration API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "vmwareengine": { - "description": "VMware Engine API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "vpcaccess": { - "description": "Serverless VPC Access API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "webrisk": { - "description": "Web Risk API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "websecurityscanner": { - "description": "Web Security Scanner API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "workflows": { - "description": "Workflow Executions API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "workstations": { - "description": "Workstations API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - } - } - } diff --git a/ports/pcre2/fix-cmake.patch b/ports/pcre2/fix-cmake.patch deleted file mode 100644 index 93d2f7196957..000000000000 --- a/ports/pcre2/fix-cmake.patch +++ /dev/null @@ -1,334 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index cec7dfb..84d1769 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -100,6 +100,9 @@ - # 2021-08-28 PH increased minimum version - # 2021-08-28 PH added test for realpath() - # 2022-12-10 PH added support for pcre2posix_test -+# 2023-01-15 Carlo added C99 as the minimum required -+# 2023-06-03 Theodore used standard CMake constructs to export the library's targets. -+# 2023-08-06 PH added support for setting variable length lookbehind maximum - - # Increased minimum to 2.8.5 to support GNUInstallDirs. - # Increased minimum to 3.1 to support imported targets. -@@ -136,6 +139,7 @@ INCLUDE(CheckFunctionExists) - INCLUDE(CheckSymbolExists) - INCLUDE(CheckIncludeFile) - INCLUDE(CheckTypeSize) -+INCLUDE(CMakePackageConfigHelpers) - INCLUDE(GNUInstallDirs) # for CMAKE_INSTALL_LIBDIR - - CHECK_INCLUDE_FILE(dirent.h HAVE_DIRENT_H) -@@ -705,7 +709,9 @@ IF(PCRE2_BUILD_PCRE2_8) - VERSION ${LIBPCRE2_8_VERSION} - SOVERSION ${LIBPCRE2_8_SOVERSION}) - TARGET_COMPILE_DEFINITIONS(pcre2-8-static PUBLIC PCRE2_STATIC) -- TARGET_INCLUDE_DIRECTORIES(pcre2-8-static PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-8-static PUBLIC -+ $ -+ $) - IF(REQUIRE_PTHREAD) - TARGET_LINK_LIBRARIES(pcre2-8-static Threads::Threads) - ENDIF(REQUIRE_PTHREAD) -@@ -718,8 +724,9 @@ IF(PCRE2_BUILD_PCRE2_8) - VERSION ${LIBPCRE2_POSIX_VERSION} - SOVERSION ${LIBPCRE2_POSIX_SOVERSION}) - TARGET_LINK_LIBRARIES(pcre2-posix-static pcre2-8-static) -- TARGET_COMPILE_DEFINITIONS(pcre2-posix-static PUBLIC PCRE2_STATIC) -- TARGET_INCLUDE_DIRECTORIES(pcre2-posix-static PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-posix-static PUBLIC -+ $ -+ $) - SET(targets ${targets} pcre2-posix-static) - - IF(MSVC) -@@ -736,7 +743,9 @@ IF(PCRE2_BUILD_PCRE2_8) - - IF(BUILD_SHARED_LIBS) - ADD_LIBRARY(pcre2-8-shared SHARED ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-8-shared PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-8-shared PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-8-shared PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_8_MACHO_COMPATIBILITY_VERSION}" -@@ -749,7 +758,9 @@ IF(PCRE2_BUILD_PCRE2_8) - ENDIF(REQUIRE_PTHREAD) - SET(targets ${targets} pcre2-8-shared) - ADD_LIBRARY(pcre2-posix-shared SHARED ${PCRE2POSIX_HEADERS} ${PCRE2POSIX_SOURCES}) -- TARGET_INCLUDE_DIRECTORIES(pcre2-posix-shared PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-posix-shared PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-posix-shared PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_POSIX_MACHO_COMPATIBILITY_VERSION}" -@@ -786,7 +797,9 @@ ENDIF(PCRE2_BUILD_PCRE2_8) - IF(PCRE2_BUILD_PCRE2_16) - IF(BUILD_STATIC_LIBS) - ADD_LIBRARY(pcre2-16-static STATIC ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-16-static PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-16-static PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=16 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" -@@ -811,7 +824,9 @@ IF(PCRE2_BUILD_PCRE2_16) - - IF(BUILD_SHARED_LIBS) - ADD_LIBRARY(pcre2-16-shared SHARED ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-16-shared PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-16-shared PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-16-shared PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=16 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" -@@ -848,7 +863,9 @@ ENDIF(PCRE2_BUILD_PCRE2_16) - IF(PCRE2_BUILD_PCRE2_32) - IF(BUILD_STATIC_LIBS) - ADD_LIBRARY(pcre2-32-static STATIC ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-32-static PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-32-static PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=32 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" -@@ -873,7 +890,9 @@ IF(PCRE2_BUILD_PCRE2_32) - - IF(BUILD_SHARED_LIBS) - ADD_LIBRARY(pcre2-32-shared SHARED ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-32-shared PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-32-shared PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-32-shared PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=32 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" -@@ -1075,9 +1094,13 @@ ENDIF(PCRE2_BUILD_TESTS) - SET(CMAKE_INSTALL_ALWAYS 1) - - INSTALL(TARGETS ${targets} -- RUNTIME DESTINATION bin -+ EXPORT pcre2-targets -+ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) -+INSTALL(EXPORT pcre2-targets -+ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcre2 -+ NAMESPACE pcre2::) - INSTALL(FILES ${pkg_config_files} DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) - INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/pcre2-config" - DESTINATION bin -@@ -1089,11 +1112,12 @@ INSTALL(FILES ${PCRE2_HEADERS} ${PCRE2POSIX_HEADERS} DESTINATION include) - # CMake config files. - set(PCRE2_CONFIG_IN ${CMAKE_CURRENT_SOURCE_DIR}/cmake/pcre2-config.cmake.in) - set(PCRE2_CONFIG_OUT ${CMAKE_CURRENT_BINARY_DIR}/cmake/pcre2-config.cmake) --configure_file(${PCRE2_CONFIG_IN} ${PCRE2_CONFIG_OUT} @ONLY) --set(PCRE2_CONFIG_VERSION_IN ${CMAKE_CURRENT_SOURCE_DIR}/cmake/pcre2-config-version.cmake.in) -+configure_package_config_file(${PCRE2_CONFIG_IN} ${PCRE2_CONFIG_OUT} INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcre2) - set(PCRE2_CONFIG_VERSION_OUT ${CMAKE_CURRENT_BINARY_DIR}/cmake/pcre2-config-version.cmake) --configure_file(${PCRE2_CONFIG_VERSION_IN} ${PCRE2_CONFIG_VERSION_OUT} @ONLY) --install(FILES ${PCRE2_CONFIG_OUT} ${PCRE2_CONFIG_VERSION_OUT} DESTINATION cmake) -+write_basic_package_version_file(${PCRE2_CONFIG_VERSION_OUT} -+ VERSION ${PCRE2_MAJOR}.${PCRE2_MINOR}.0 -+ COMPATIBILITY SameMajorVersion) -+install(FILES ${PCRE2_CONFIG_OUT} ${PCRE2_CONFIG_VERSION_OUT} DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcre2) - - FILE(GLOB html ${PROJECT_SOURCE_DIR}/doc/html/*.html) - FILE(GLOB man1 ${PROJECT_SOURCE_DIR}/doc/*.1) -diff --git a/cmake/pcre2-config-version.cmake.in b/cmake/pcre2-config-version.cmake.in -deleted file mode 100644 -index dac149e..0000000 ---- a/cmake/pcre2-config-version.cmake.in -+++ /dev/null -@@ -1,15 +0,0 @@ --set(PACKAGE_VERSION_MAJOR @PCRE2_MAJOR@) --set(PACKAGE_VERSION_MINOR @PCRE2_MINOR@) --set(PACKAGE_VERSION_PATCH 0) --set(PACKAGE_VERSION @PCRE2_MAJOR@.@PCRE2_MINOR@.0) -- --# Check whether the requested PACKAGE_FIND_VERSION is compatible --if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION OR -- PACKAGE_VERSION_MAJOR GREATER PACKAGE_FIND_VERSION_MAJOR) -- set(PACKAGE_VERSION_COMPATIBLE FALSE) --else() -- set(PACKAGE_VERSION_COMPATIBLE TRUE) -- if(PACKAGE_VERSION VERSION_EQUAL PACKAGE_FIND_VERSION) -- set(PACKAGE_VERSION_EXACT TRUE) -- endif() --endif() -diff --git a/cmake/pcre2-config.cmake.in b/cmake/pcre2-config.cmake.in -index b313d6d..159669b 100644 ---- a/cmake/pcre2-config.cmake.in -+++ b/cmake/pcre2-config.cmake.in -@@ -5,11 +5,17 @@ - # - # Static vs. shared - # ----------------- --# To make use of the static library instead of the shared one, one needs -+# To force using the static library instead of the shared one, one needs - # to set the variable PCRE2_USE_STATIC_LIBS to ON before calling find_package. -+# If the variable is not set, the static library will be used if only that has -+# been built, otherwise the shared library will be used. -+# -+# The following components are supported: 8BIT, 16BIT, 32BIT and POSIX. -+# They used to be required but not anymore; all available targets will -+# be defined regardless of the requested components. - # Example: - # set(PCRE2_USE_STATIC_LIBS ON) --# find_package(PCRE2 CONFIG COMPONENTS 8BIT) -+# find_package(PCRE2 CONFIG) - # - # This will define the following variables: - # -@@ -23,70 +29,42 @@ - # PCRE2::32BIT - The 32 bit PCRE2 library. - # PCRE2::POSIX - The POSIX PCRE2 library. - --set(PCRE2_NON_STANDARD_LIB_PREFIX @NON_STANDARD_LIB_PREFIX@) --set(PCRE2_NON_STANDARD_LIB_SUFFIX @NON_STANDARD_LIB_SUFFIX@) --set(PCRE2_8BIT_NAME pcre2-8) --set(PCRE2_16BIT_NAME pcre2-16) --set(PCRE2_32BIT_NAME pcre2-32) --set(PCRE2_POSIX_NAME pcre2-posix) --find_path(PCRE2_INCLUDE_DIR NAMES pcre2.h DOC "PCRE2 include directory") --if (PCRE2_USE_STATIC_LIBS) -- if (MSVC) -- set(PCRE2_8BIT_NAME pcre2-8-static) -- set(PCRE2_16BIT_NAME pcre2-16-static) -- set(PCRE2_32BIT_NAME pcre2-32-static) -- set(PCRE2_POSIX_NAME pcre2-posix-static) -- endif () -+@PACKAGE_INIT@ - -- set(PCRE2_PREFIX ${CMAKE_STATIC_LIBRARY_PREFIX}) -- set(PCRE2_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX}) --else () -- set(PCRE2_PREFIX ${CMAKE_SHARED_LIBRARY_PREFIX}) -- if (MINGW AND PCRE2_NON_STANDARD_LIB_PREFIX) -- set(PCRE2_PREFIX "") -- endif () -+include(CMakeFindDependencyMacro) -+if("@REQUIRE_PTHREAD@") # REQUIRE_PTHREAD -+ find_dependency(Threads) -+endif() - -- set(PCRE2_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX}) -- if (MINGW AND PCRE2_NON_STANDARD_LIB_SUFFIX) -- set(PCRE2_SUFFIX "-0.dll") -- endif () --endif () --find_library(PCRE2_8BIT_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "8 bit PCRE2 library") --find_library(PCRE2_16BIT_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_16BIT_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "16 bit PCRE2 library") --find_library(PCRE2_32BIT_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_32BIT_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "32 bit PCRE2 library") --find_library(PCRE2_POSIX_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_POSIX_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "8 bit POSIX PCRE2 library") --unset(PCRE2_NON_STANDARD_LIB_PREFIX) --unset(PCRE2_NON_STANDARD_LIB_SUFFIX) --unset(PCRE2_8BIT_NAME) --unset(PCRE2_16BIT_NAME) --unset(PCRE2_32BIT_NAME) --unset(PCRE2_POSIX_NAME) -+include("${CMAKE_CURRENT_LIST_DIR}/pcre2-targets.cmake") - - # Set version --if (PCRE2_INCLUDE_DIR) -- set(PCRE2_VERSION "@PCRE2_MAJOR@.@PCRE2_MINOR@.0") --endif () -+set(PCRE2_VERSION "@PCRE2_MAJOR@.@PCRE2_MINOR@.0") - --# Which components have been found. --if (PCRE2_8BIT_LIBRARY) -- set(PCRE2_8BIT_FOUND TRUE) --endif () --if (PCRE2_16BIT_LIBRARY) -- set(PCRE2_16BIT_FOUND TRUE) --endif () --if (PCRE2_32BIT_LIBRARY) -- set(PCRE2_32BIT_FOUND TRUE) --endif () --if (PCRE2_POSIX_LIBRARY) -- set(PCRE2_POSIX_FOUND TRUE) --endif () -- --# Check if at least one component has been specified. --list(LENGTH PCRE2_FIND_COMPONENTS PCRE2_NCOMPONENTS) --if (PCRE2_NCOMPONENTS LESS 1) -- message(FATAL_ERROR "No components have been specified. This is not allowed. Please, specify at least one component.") --endif () --unset(PCRE2_NCOMPONENTS) -+# Chooses the linkage of the library to expose in the -+# unsuffixed edition of the target. -+macro(_pcre2_add_component_target component target) -+ # If the static library exists and either PCRE2_USE_STATIC_LIBS -+ # is defined, or the dynamic library does not exist, use the static library. -+ if(NOT TARGET PCRE2::${component}) -+ if(TARGET pcre2::pcre2-${target}-static AND (PCRE2_USE_STATIC_LIBS OR NOT TARGET pcre2::pcre2-${target}-shared)) -+ add_library(PCRE2::${component} ALIAS pcre2::pcre2-${target}-static) -+ set(PCRE2_${component}_FOUND TRUE) -+ # Otherwise use the dynamic library if it exists. -+ elseif(TARGET pcre2::pcre2-${target}-shared AND NOT PCRE2_USE_STATIC_LIBS) -+ add_library(PCRE2::${component} ALIAS pcre2::pcre2-${target}-shared) -+ set(PCRE2_${component}_FOUND TRUE) -+ endif() -+ if(PCRE2_${component}_FOUND) -+ get_target_property(PCRE2_${component}_LIBRARY PCRE2::${component} IMPORTED_LOCATION) -+ set(PCRE2_LIBRARIES ${PCRE2_LIBRARIES} ${PCRE2_${component}_LIBRARY}) -+ endif() -+ endif() -+endmacro() -+_pcre2_add_component_target(8BIT 8) -+_pcre2_add_component_target(16BIT 16) -+_pcre2_add_component_target(32BIT 32) -+_pcre2_add_component_target(POSIX posix) - - # When POSIX component has been specified make sure that also 8BIT component is specified. - set(PCRE2_8BIT_COMPONENT FALSE) -@@ -105,41 +83,5 @@ endif() - unset(PCRE2_8BIT_COMPONENT) - unset(PCRE2_POSIX_COMPONENT) - --include(FindPackageHandleStandardArgs) --set(${CMAKE_FIND_PACKAGE_NAME}_CONFIG "${CMAKE_CURRENT_LIST_FILE}") --find_package_handle_standard_args(PCRE2 -- FOUND_VAR PCRE2_FOUND -- REQUIRED_VARS PCRE2_INCLUDE_DIR -- HANDLE_COMPONENTS -- VERSION_VAR PCRE2_VERSION -- CONFIG_MODE --) -- --set(PCRE2_LIBRARIES) --if (PCRE2_FOUND) -- foreach(component ${PCRE2_FIND_COMPONENTS}) -- if (PCRE2_USE_STATIC_LIBS) -- add_library(PCRE2::${component} STATIC IMPORTED) -- target_compile_definitions(PCRE2::${component} INTERFACE PCRE2_STATIC) -- else () -- add_library(PCRE2::${component} SHARED IMPORTED) -- endif () -- set_target_properties(PCRE2::${component} PROPERTIES -- IMPORTED_LOCATION "${PCRE2_${component}_LIBRARY}" -- INTERFACE_INCLUDE_DIRECTORIES "${PCRE2_INCLUDE_DIR}" -- ) -- if (component STREQUAL "POSIX") -- set_target_properties(PCRE2::${component} PROPERTIES -- INTERFACE_LINK_LIBRARIES "PCRE2::8BIT" -- LINK_LIBRARIES "PCRE2::8BIT" -- ) -- endif () -- -- set(PCRE2_LIBRARIES ${PCRE2_LIBRARIES} ${PCRE2_${component}_LIBRARY}) -- mark_as_advanced(PCRE2_${component}_LIBRARY) -- endforeach() --endif () -- --mark_as_advanced( -- PCRE2_INCLUDE_DIR --) -+# Check for required components. -+check_required_components("PCRE2") diff --git a/ports/pcre2/no-static-suffix.patch b/ports/pcre2/no-static-suffix.patch deleted file mode 100644 index 7f41bcd566cf..000000000000 --- a/ports/pcre2/no-static-suffix.patch +++ /dev/null @@ -1,33 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index fa2181e..3bf5317 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -711,8 +711,8 @@ IF(PCRE2_BUILD_PCRE2_8) - SET(targets ${targets} pcre2-posix-static) - - IF(MSVC) -- SET_TARGET_PROPERTIES(pcre2-8-static PROPERTIES OUTPUT_NAME pcre2-8-static) -- SET_TARGET_PROPERTIES(pcre2-posix-static PROPERTIES OUTPUT_NAME pcre2-posix-static) -+ SET_TARGET_PROPERTIES(pcre2-8-static PROPERTIES OUTPUT_NAME pcre2-8) -+ SET_TARGET_PROPERTIES(pcre2-posix-static PROPERTIES OUTPUT_NAME pcre2-posix) - ELSE(MSVC) - SET_TARGET_PROPERTIES(pcre2-8-static PROPERTIES OUTPUT_NAME pcre2-8) - SET_TARGET_PROPERTIES(pcre2-posix-static PROPERTIES OUTPUT_NAME pcre2-posix) -@@ -777,7 +777,7 @@ IF(PCRE2_BUILD_PCRE2_16) - SET(targets ${targets} pcre2-16-static) - - IF(MSVC) -- SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES OUTPUT_NAME pcre2-16-static) -+ SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES OUTPUT_NAME pcre2-16) - ELSE(MSVC) - SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES OUTPUT_NAME pcre2-16) - ENDIF(MSVC) -@@ -829,7 +829,7 @@ IF(PCRE2_BUILD_PCRE2_32) - SET(targets ${targets} pcre2-32-static) - - IF(MSVC) -- SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES OUTPUT_NAME pcre2-32-static) -+ SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES OUTPUT_NAME pcre2-32) - ELSE(MSVC) - SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES OUTPUT_NAME pcre2-32) - ENDIF(MSVC) diff --git a/ports/pcre2/pcre2-10.35_fix-uwp.patch b/ports/pcre2/pcre2-10.35_fix-uwp.patch deleted file mode 100644 index 476dde0f6a4c..000000000000 --- a/ports/pcre2/pcre2-10.35_fix-uwp.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- a/CMakeLists.txt 2020-05-09 16:43:10.000000000 +0200 -+++ b/CMakeLists.txt 2020-06-03 20:57:17.026182500 +0200 -@@ -619,6 +619,7 @@ - - IF(MSVC) - ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS) -+ add_compile_options(/wd4146) - ENDIF(MSVC) - - SET(CMAKE_INCLUDE_CURRENT_DIR 1) diff --git a/ports/pcre2/portfile.cmake b/ports/pcre2/portfile.cmake deleted file mode 100644 index 5d6c5c39b6a6..000000000000 --- a/ports/pcre2/portfile.cmake +++ /dev/null @@ -1,73 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO PCRE2Project/pcre2 - REF "pcre2-${VERSION}" - SHA512 3d0ee66e23809d3da2fe2bf4ed6e20b0fb96c293a91668935f6319e8d02e480eeef33da01e08a7436a18a1a85a116d83186b953520f394c866aad3cea73c7f5c - HEAD_REF master - PATCHES - pcre2-10.35_fix-uwp.patch - no-static-suffix.patch - fix-cmake.patch -) - -string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" BUILD_STATIC) -string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" INSTALL_PDB) -string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "static" BUILD_STATIC_CRT) - -vcpkg_check_features( - OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - jit PCRE2_SUPPORT_JIT -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - ${FEATURE_OPTIONS} - -DBUILD_STATIC_LIBS=${BUILD_STATIC} - -DPCRE2_STATIC_RUNTIME=${BUILD_STATIC_CRT} - -DPCRE2_BUILD_PCRE2_8=ON - -DPCRE2_BUILD_PCRE2_16=ON - -DPCRE2_BUILD_PCRE2_32=ON - -DPCRE2_SUPPORT_UNICODE=ON - -DPCRE2_BUILD_TESTS=OFF - -DPCRE2_BUILD_PCRE2GREP=OFF - -DCMAKE_DISABLE_FIND_PACKAGE_BZip2=ON - -DCMAKE_DISABLE_FIND_PACKAGE_ZLIB=ON - -DCMAKE_DISABLE_FIND_PACKAGE_Readline=ON - -DCMAKE_DISABLE_FIND_PACKAGE_Editline=ON - -DINSTALL_MSVC_PDB=${INSTALL_PDB} - ) - -vcpkg_cmake_install() -vcpkg_copy_pdbs() - -file(READ "${CURRENT_PACKAGES_DIR}/include/pcre2.h" PCRE2_H) -if(BUILD_STATIC) - string(REPLACE "defined(PCRE2_STATIC)" "1" PCRE2_H "${PCRE2_H}") -else() - string(REPLACE "defined(PCRE2_STATIC)" "0" PCRE2_H "${PCRE2_H}") -endif() -file(WRITE "${CURRENT_PACKAGES_DIR}/include/pcre2.h" "${PCRE2_H}") - -vcpkg_fixup_pkgconfig() -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/${PORT}) - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/man" - "${CURRENT_PACKAGES_DIR}/share/doc" - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/man" - "${CURRENT_PACKAGES_DIR}/debug/share") - -if(BUILD_STATIC) - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") -elseif(VCPKG_TARGET_IS_WINDOWS) - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/bin/pcre2-config" "${CURRENT_PACKAGES_DIR}" "`dirname $0`/..") - if(EXISTS "${CURRENT_PACKAGES_DIR}/debug/bin/pcre2-config") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/bin/pcre2-config" "${CURRENT_PACKAGES_DIR}" "`dirname $0`/../..") - endif() -endif() - -file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") diff --git a/ports/pcre2/usage b/ports/pcre2/usage deleted file mode 100644 index a8e97871bd52..000000000000 --- a/ports/pcre2/usage +++ /dev/null @@ -1,6 +0,0 @@ -The package pcre2 is compatible with built-in CMake targets: - - # Each component imports a target: - # TARGETS: pcre2::8BIT pcre2::16BIT pcre2::32BIT pcre2::POSIX - find_package(pcre2 CONFIG REQUIRED) - target_link_libraries(main PRIVATE pcre2::8BIT pcre2::POSIX) diff --git a/ports/pcre2/vcpkg.json b/ports/pcre2/vcpkg.json deleted file mode 100644 index 3dd6b9455d55..000000000000 --- a/ports/pcre2/vcpkg.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "pcre2", - "version": "10.42", - "port-version": 1, - "description": "Regular Expression pattern matching using the same syntax and semantics as Perl 5.", - "homepage": "https://github.com/PCRE2Project/pcre2", - "license": "BSD-3-Clause", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "platform-default-features" - ], - "features": { - "jit": { - "description": "Enable support for Just-In-Time compiling regex matchers", - "supports": "!emscripten" - }, - "platform-default-features": { - "description": "Enable default features", - "dependencies": [ - { - "name": "pcre2", - "features": [ - "jit" - ], - "platform": "!emscripten" - } - ] - } - } -} diff --git a/ports/triplets/arm64-osx-asan.cmake b/ports/triplets/arm64-osx-asan.cmake index 4aa2d22af543..d9980f093784 100644 --- a/ports/triplets/arm64-osx-asan.cmake +++ b/ports/triplets/arm64-osx-asan.cmake @@ -7,3 +7,9 @@ set(VCPKG_OSX_ARCHITECTURES arm64) set(VCPKG_C_FLAGS "-fsanitize=address") set(VCPKG_CXX_FLAGS "-fsanitize=address") + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/arm64-osx-release.cmake b/ports/triplets/arm64-osx-release.cmake index 0f0d0b2f20b5..0b02711121d3 100644 --- a/ports/triplets/arm64-osx-release.cmake +++ b/ports/triplets/arm64-osx-release.cmake @@ -7,3 +7,9 @@ set(VCPKG_OSX_ARCHITECTURES arm64) set(VCPKG_OSX_DEPLOYMENT_TARGET 11) set(VCPKG_BUILD_TYPE release) + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/arm64-osx-relwithdebinfo.cmake b/ports/triplets/arm64-osx-relwithdebinfo.cmake index 592c7f2ae832..36b81fa41acb 100644 --- a/ports/triplets/arm64-osx-relwithdebinfo.cmake +++ b/ports/triplets/arm64-osx-relwithdebinfo.cmake @@ -8,3 +8,9 @@ set(VCPKG_OSX_DEPLOYMENT_TARGET 11) set(VCPKG_CXX_FLAGS "-g") set(VCPKG_C_FLAGS "-g") + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/arm64-osx.cmake b/ports/triplets/arm64-osx.cmake index 348949babcfb..c41b4ea47aff 100644 --- a/ports/triplets/arm64-osx.cmake +++ b/ports/triplets/arm64-osx.cmake @@ -5,3 +5,9 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES arm64) set(VCPKG_OSX_DEPLOYMENT_TARGET 11) + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/x64-linux-asan.cmake b/ports/triplets/x64-linux-asan.cmake index 747d7d620207..bb3ce9143d8e 100644 --- a/ports/triplets/x64-linux-asan.cmake +++ b/ports/triplets/x64-linux-asan.cmake @@ -6,3 +6,9 @@ set(VCPKG_CMAKE_SYSTEM_NAME Linux) set(VCPKG_C_FLAGS "-fsanitize=address") set(VCPKG_CXX_FLAGS "-fsanitize=address") + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/x64-linux-release.cmake b/ports/triplets/x64-linux-release.cmake index 8a25c72e1e89..7db4de766ae1 100644 --- a/ports/triplets/x64-linux-release.cmake +++ b/ports/triplets/x64-linux-release.cmake @@ -5,3 +5,9 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Linux) set(VCPKG_BUILD_TYPE release) + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/x64-linux-relwithdebinfo.cmake b/ports/triplets/x64-linux-relwithdebinfo.cmake index 533c1f3928cb..4bde165e3b9a 100644 --- a/ports/triplets/x64-linux-relwithdebinfo.cmake +++ b/ports/triplets/x64-linux-relwithdebinfo.cmake @@ -6,3 +6,9 @@ set(VCPKG_CMAKE_SYSTEM_NAME Linux) set(VCPKG_CXX_FLAGS "-g") set(VCPKG_C_FLAGS "-g") + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/x64-linux.cmake b/ports/triplets/x64-linux.cmake index ed59697f3117..96a76c50a15d 100644 --- a/ports/triplets/x64-linux.cmake +++ b/ports/triplets/x64-linux.cmake @@ -7,3 +7,9 @@ set(VCPKG_CMAKE_SYSTEM_NAME Linux) # For AWS SDK. Remove after update from 1.74 set(VCPKG_CXX_FLAGS "-Wno-error=nonnull -Wno-error=deprecated-declarations") set(VCPKG_C_FLAGS "-Wno-error=nonnull -Wno-error=deprecated-declarations") + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/x64-osx-asan.cmake b/ports/triplets/x64-osx-asan.cmake index aac0688e2c7c..f4d7c1416439 100644 --- a/ports/triplets/x64-osx-asan.cmake +++ b/ports/triplets/x64-osx-asan.cmake @@ -7,3 +7,9 @@ set(VCPKG_OSX_ARCHITECTURES x86_64) set(VCPKG_C_FLAGS "-fsanitize=address") set(VCPKG_CXX_FLAGS "-fsanitize=address") + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/x64-osx-release.cmake b/ports/triplets/x64-osx-release.cmake index 461a5a727ec9..81d64e4b1f0a 100644 --- a/ports/triplets/x64-osx-release.cmake +++ b/ports/triplets/x64-osx-release.cmake @@ -7,3 +7,9 @@ set(VCPKG_OSX_ARCHITECTURES x86_64) set(VCPKG_OSX_DEPLOYMENT_TARGET 11) set(VCPKG_BUILD_TYPE release) + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/x64-osx-relwithdebinfo.cmake b/ports/triplets/x64-osx-relwithdebinfo.cmake index db1bfabf76d9..d22e1e1d6e12 100644 --- a/ports/triplets/x64-osx-relwithdebinfo.cmake +++ b/ports/triplets/x64-osx-relwithdebinfo.cmake @@ -8,3 +8,9 @@ set(VCPKG_OSX_DEPLOYMENT_TARGET 11) set(VCPKG_CXX_FLAGS "-g") set(VCPKG_C_FLAGS "-g") + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/triplets/x64-osx.cmake b/ports/triplets/x64-osx.cmake index e8a4c8d9d69e..b77361c57d0c 100644 --- a/ports/triplets/x64-osx.cmake +++ b/ports/triplets/x64-osx.cmake @@ -5,3 +5,9 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES x86_64) set(VCPKG_OSX_DEPLOYMENT_TARGET 11) + +# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries +# like arrow (https://github.com/apache/arrow/issues/42154). +if("${PORT}" MATCHES "^aws-") + set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden") +endif() diff --git a/ports/vcpkg-cmake-config/copyright b/ports/vcpkg-cmake-config/copyright deleted file mode 100644 index 2e4eac8264fa..000000000000 --- a/ports/vcpkg-cmake-config/copyright +++ /dev/null @@ -1,23 +0,0 @@ -Copyright (c) Microsoft Corporation - -All rights reserved. - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/ports/vcpkg-cmake-config/portfile.cmake b/ports/vcpkg-cmake-config/portfile.cmake deleted file mode 100644 index fc3dbafd5184..000000000000 --- a/ports/vcpkg-cmake-config/portfile.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT TARGET_TRIPLET STREQUAL _HOST_TRIPLET) - # make FATAL_ERROR in CI when issue #16773 fixed - message(WARNING "vcpkg-cmake-config is a host-only port; please mark it as a host port in your dependencies.") -endif() - -file(INSTALL - "${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_config_fixup.cmake" - "${CMAKE_CURRENT_LIST_DIR}/vcpkg-port-config.cmake" - "${CMAKE_CURRENT_LIST_DIR}/copyright" - DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") - -set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/ports/vcpkg-cmake-config/vcpkg-port-config.cmake b/ports/vcpkg-cmake-config/vcpkg-port-config.cmake deleted file mode 100644 index 980d411315c7..000000000000 --- a/ports/vcpkg-cmake-config/vcpkg-port-config.cmake +++ /dev/null @@ -1 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_config_fixup.cmake") diff --git a/ports/vcpkg-cmake-config/vcpkg.json b/ports/vcpkg-cmake-config/vcpkg.json deleted file mode 100644 index df1da76c8f70..000000000000 --- a/ports/vcpkg-cmake-config/vcpkg.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "vcpkg-cmake-config", - "version-date": "2024-04-07", - "documentation": "https://vcpkg.io/en/docs/README.html", - "license": "MIT" -} diff --git a/ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake b/ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake deleted file mode 100644 index 246588129a91..000000000000 --- a/ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake +++ /dev/null @@ -1,266 +0,0 @@ -include_guard(GLOBAL) - -function(vcpkg_cmake_config_fixup) - cmake_parse_arguments(PARSE_ARGV 0 "arg" "DO_NOT_DELETE_PARENT_CONFIG_PATH;NO_PREFIX_CORRECTION" "PACKAGE_NAME;CONFIG_PATH;TOOLS_PATH" "") - - if(DEFINED arg_UNPARSED_ARGUMENTS) - message(FATAL_ERROR "vcpkg_cmake_config_fixup was passed extra arguments: ${arg_UNPARSED_ARGUMENTS}") - endif() - if(NOT arg_PACKAGE_NAME) - set(arg_PACKAGE_NAME "${PORT}") - endif() - if(NOT arg_CONFIG_PATH) - set(arg_CONFIG_PATH "share/${arg_PACKAGE_NAME}") - endif() - if(NOT arg_TOOLS_PATH) - set(arg_TOOLS_PATH "tools/${PORT}") - endif() - set(target_path "share/${arg_PACKAGE_NAME}") - - string(REPLACE "." "\\." EXECUTABLE_SUFFIX "${VCPKG_TARGET_EXECUTABLE_SUFFIX}") - - set(debug_share "${CURRENT_PACKAGES_DIR}/debug/${target_path}") - set(release_share "${CURRENT_PACKAGES_DIR}/${target_path}") - - if(NOT arg_CONFIG_PATH STREQUAL "share/${arg_PACKAGE_NAME}") - if(arg_CONFIG_PATH STREQUAL "share") - set(arg_CONFIG_PATH z_vcpkg_share) - file(RENAME "${CURRENT_PACKAGES_DIR}/debug/share" "${CURRENT_PACKAGES_DIR}/debug/${arg_CONFIG_PATH}") - file(RENAME "${CURRENT_PACKAGES_DIR}/share" "${CURRENT_PACKAGES_DIR}/${arg_CONFIG_PATH}") - endif() - - set(debug_config "${CURRENT_PACKAGES_DIR}/debug/${arg_CONFIG_PATH}") - set(release_config "${CURRENT_PACKAGES_DIR}/${arg_CONFIG_PATH}") - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - if(NOT EXISTS "${debug_config}") - message(FATAL_ERROR "'${debug_config}' does not exist.") - endif() - - # This roundabout handling enables CONFIG_PATH = share - file(MAKE_DIRECTORY "${debug_share}") - file(GLOB files "${debug_config}/*") - file(COPY ${files} DESTINATION "${debug_share}") - file(REMOVE_RECURSE "${debug_config}") - endif() - - file(GLOB files "${release_config}/*") - file(COPY ${files} DESTINATION "${release_share}") - file(REMOVE_RECURSE "${release_config}") - - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - get_filename_component(debug_config_dir_name "${debug_config}" NAME) - string(TOLOWER "${debug_config_dir_name}" debug_config_dir_name) - if(debug_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) - file(REMOVE_RECURSE "${debug_config}") - else() - get_filename_component(debug_config_parent_dir "${debug_config}" DIRECTORY) - get_filename_component(debug_config_dir_name "${debug_config_parent_dir}" NAME) - string(TOLOWER "${debug_config_dir_name}" debug_config_dir_name) - if(debug_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) - file(REMOVE_RECURSE "${debug_config_parent_dir}") - endif() - endif() - endif() - - get_filename_component(release_config_dir_name "${release_config}" NAME) - string(TOLOWER "${release_config_dir_name}" release_config_dir_name) - if(release_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) - file(REMOVE_RECURSE "${release_config}") - else() - get_filename_component(release_config_parent_dir "${release_config}" DIRECTORY) - get_filename_component(release_config_dir_name "${release_config_parent_dir}" NAME) - string(TOLOWER "${release_config_dir_name}" release_config_dir_name) - if(release_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) - file(REMOVE_RECURSE "${release_config_parent_dir}") - endif() - endif() - endif() - - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - if(NOT EXISTS "${debug_share}") - message(FATAL_ERROR "'${debug_share}' does not exist.") - endif() - endif() - - file(GLOB_RECURSE release_targets - "${release_share}/*-release.cmake" - ) - foreach(release_target IN LISTS release_targets) - file(READ "${release_target}" contents) - string(REPLACE "${CURRENT_INSTALLED_DIR}" "\${_IMPORT_PREFIX}" contents "${contents}") - string(REGEX REPLACE "\\\${_IMPORT_PREFIX}/bin/([^ \"]+${EXECUTABLE_SUFFIX})" "\${_IMPORT_PREFIX}/${arg_TOOLS_PATH}/\\1" contents "${contents}") - file(WRITE "${release_target}" "${contents}") - endforeach() - - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(GLOB_RECURSE debug_targets - "${debug_share}/*-debug.cmake" - ) - foreach(debug_target IN LISTS debug_targets) - file(RELATIVE_PATH debug_target_rel "${debug_share}" "${debug_target}") - - file(READ "${debug_target}" contents) - string(REPLACE "${CURRENT_INSTALLED_DIR}" "\${_IMPORT_PREFIX}" contents "${contents}") - string(REGEX REPLACE "\\\${_IMPORT_PREFIX}/bin/([^ \";]+${EXECUTABLE_SUFFIX})" "\${_IMPORT_PREFIX}/${arg_TOOLS_PATH}/\\1" contents "${contents}") - string(REPLACE "\${_IMPORT_PREFIX}/lib" "\${_IMPORT_PREFIX}/debug/lib" contents "${contents}") - string(REPLACE "\${_IMPORT_PREFIX}/bin" "\${_IMPORT_PREFIX}/debug/bin" contents "${contents}") - file(WRITE "${release_share}/${debug_target_rel}" "${contents}") - - file(REMOVE "${debug_target}") - endforeach() - endif() - - #Fix ${_IMPORT_PREFIX} and absolute paths in cmake generated targets and configs; - #Since those can be renamed we have to check in every *.cmake, but only once. - file(GLOB_RECURSE main_cmakes "${release_share}/*.cmake") - if(NOT DEFINED Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP) - vcpkg_list(SET Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP) - endif() - foreach(already_fixed_up IN LISTS Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP) - vcpkg_list(REMOVE_ITEM main_cmakes "${already_fixed_up}") - endforeach() - vcpkg_list(APPEND Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP ${main_cmakes}) - set(Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP "${Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP}" CACHE INTERNAL "") - - foreach(main_cmake IN LISTS main_cmakes) - file(READ "${main_cmake}" contents) - # Note: I think the following comment is no longer true, since we now require the path to be `share/blah` - # however, I don't know it for sure. - # - nimazzuc - - #This correction is not correct for all cases. To make it correct for all cases it needs to consider - #original folder deepness to CURRENT_PACKAGES_DIR in comparison to the moved to folder deepness which - #is always at least (>=) 2, e.g. share/${PORT}. Currently the code assumes it is always 2 although - #this requirement is only true for the *Config.cmake. The targets are not required to be in the same - #folder as the *Config.cmake! - if(NOT arg_NO_PREFIX_CORRECTION) - string(REGEX REPLACE -[[get_filename_component\(_IMPORT_PREFIX "\${CMAKE_CURRENT_LIST_FILE}" PATH\)( -get_filename_component\(_IMPORT_PREFIX "\${_IMPORT_PREFIX}" PATH\))*]] -[[get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) -get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) -get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH)]] - contents "${contents}") # see #1044 for details why this replacement is necessary. See #4782 why it must be a regex. - string(REGEX REPLACE -[[get_filename_component\(PACKAGE_PREFIX_DIR "\${CMAKE_CURRENT_LIST_DIR}/\.\./(\.\./)*" ABSOLUTE\)]] -[[get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../" ABSOLUTE)]] - contents "${contents}") - string(REGEX REPLACE -[[(get_filename_component\(PACKAGE_\${CMAKE_FIND_PACKAGE_NAME}_COUNTER_[1-9][0-9]*) "\${CMAKE_CURRENT_LIST_DIR}/\.\./(\.\./)*" ABSOLUTE\)]] -"\\1 \"\${CMAKE_CURRENT_LIST_DIR}/../../\" ABSOLUTE)" # adapt to https://gitlab.kitware.com/cmake/cmake/-/merge_requests/9390 - contents "${contents}") - string(REGEX REPLACE -[[get_filename_component\(PACKAGE_PREFIX_DIR "\${CMAKE_CURRENT_LIST_DIR}/\.\.((\\|/)\.\.)*" ABSOLUTE\)]] -[[get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../" ABSOLUTE)]] - contents "${contents}") # This is a meson-related workaround, see https://github.com/mesonbuild/meson/issues/6955 - string(REGEX REPLACE -[[(get_filename_component\(PACKAGE_\${CMAKE_FIND_PACKAGE_NAME}_COUNTER_[1-9][0-9]*) "\${CMAKE_CURRENT_LIST_DIR}/\.\.((\\|/)\.\.)*" ABSOLUTE\)]] -"\\1 \"\${CMAKE_CURRENT_LIST_DIR}/../../\" ABSOLUTE)" - contents "${contents}") - endif() - - # Merge release and debug configurations of target property INTERFACE_LINK_LIBRARIES. - string(REPLACE "${release_share}/" "${debug_share}/" debug_cmake "${main_cmake}") - if(DEFINED VCPKG_BUILD_TYPE) - # Skip. Warning: A release-only port in a dual-config installation - # may pull release dependencies into the debug configuration. - elseif(NOT contents MATCHES "INTERFACE_LINK_LIBRARIES") - # Skip. No relevant properties. - elseif(NOT contents MATCHES "# Generated CMake target import file\\.") - # Skip. No safe assumptions about a matching debug import file. - elseif(NOT EXISTS "${debug_cmake}") - message(SEND_ERROR "Did not find a debug import file matching '${main_cmake}'") - else() - file(READ "${debug_cmake}" debug_contents) - while(contents MATCHES "set_target_properties\\(([^ \$]*) PROPERTIES[^)]*\\)") - set(matched_command "${CMAKE_MATCH_0}") - string(REPLACE "+" "\\+" target "${CMAKE_MATCH_1}") - if(NOT debug_contents MATCHES "set_target_properties\\(${target} PROPERTIES[^)]*\\)") - message(SEND_ERROR "Did not find a debug configuration for target '${target}'.") - endif() - set(debug_command "${CMAKE_MATCH_0}") - string(REGEX MATCH " INTERFACE_LINK_LIBRARIES \"([^\"]*)\"" release_line "${matched_command}") - set(release_libs "${CMAKE_MATCH_1}") - string(REGEX MATCH " INTERFACE_LINK_LIBRARIES \"([^\"]*)\"" debug_line "${debug_command}") - set(debug_libs "${CMAKE_MATCH_1}") - z_vcpkg_cmake_config_fixup_merge(merged_libs release_libs debug_libs) - string(REPLACE "${release_line}" " INTERFACE_LINK_LIBRARIES \"${merged_libs}\"" updated_command "${matched_command}") - string(REPLACE "set_target_properties" "set_target_properties::done" updated_command "${updated_command}") # Prevend 2nd match - string(REPLACE "${matched_command}" "${updated_command}" contents "${contents}") - endwhile() - string(REPLACE "set_target_properties::done" "set_target_properties" contents "${contents}") # Restore original command - endif() - - #Fix absolute paths to installed dir with ones relative to ${CMAKE_CURRENT_LIST_DIR} - #This happens if vcpkg built libraries are directly linked to a target instead of using - #an imported target. - string(REPLACE "${CURRENT_INSTALLED_DIR}" [[${VCPKG_IMPORT_PREFIX}]] contents "${contents}") - file(TO_CMAKE_PATH "${CURRENT_PACKAGES_DIR}" cmake_current_packages_dir) - string(REPLACE "${cmake_current_packages_dir}" [[${VCPKG_IMPORT_PREFIX}]] contents "${contents}") - # If ${VCPKG_IMPORT_PREFIX} was actually used, inject a definition of it: - string(FIND "${contents}" [[${VCPKG_IMPORT_PREFIX}]] index) - if (NOT index STREQUAL "-1") - get_filename_component(main_cmake_dir "${main_cmake}" DIRECTORY) - # Calculate relative to be a sequence of "../" - file(RELATIVE_PATH relative "${main_cmake_dir}" "${cmake_current_packages_dir}") - string(PREPEND contents "get_filename_component(VCPKG_IMPORT_PREFIX \"\${CMAKE_CURRENT_LIST_DIR}\/${relative}\" ABSOLUTE)\n") - endif() - - file(WRITE "${main_cmake}" "${contents}") - endforeach() - - file(GLOB_RECURSE unused_files - "${debug_share}/*[Tt]argets.cmake" - "${debug_share}/*[Cc]onfig.cmake" - "${debug_share}/*[Cc]onfigVersion.cmake" - "${debug_share}/*[Cc]onfig-version.cmake" - ) - foreach(unused_file IN LISTS unused_files) - file(REMOVE "${unused_file}") - endforeach() - - # Remove /debug// if it's empty. - file(GLOB_RECURSE remaining_files "${debug_share}/*") - if(remaining_files STREQUAL "") - file(REMOVE_RECURSE "${debug_share}") - endif() - - # Remove /debug/share/ if it's empty. - file(GLOB_RECURSE remaining_files "${CURRENT_PACKAGES_DIR}/debug/share/*") - if(remaining_files STREQUAL "") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - endif() -endfunction() - -# Merges link interface library lists for release and debug -# into a single expression which use generator expression as necessary. -function(z_vcpkg_cmake_config_fixup_merge out_var release_var debug_var) - set(release_libs "VCPKG;${${release_var}}") - string(REGEX REPLACE ";optimized;([^;]*)" ";\\1" release_libs "${release_libs}") - string(REGEX REPLACE ";debug;([^;]*)" ";" release_libs "${release_libs}") - list(REMOVE_AT release_libs 0) - list(FILTER release_libs EXCLUDE REGEX [[^\\[$]<\\[$]:]]) - list(TRANSFORM release_libs REPLACE [[^\\[$]<\\[$]>:(.*)>$]] "\\1") - - set(debug_libs "VCPKG;${${debug_var}}") - string(REGEX REPLACE ";optimized;([^;]*)" ";" debug_libs "${debug_libs}") - string(REGEX REPLACE ";debug;([^;]*)" ";\\1" debug_libs "${debug_libs}") - list(REMOVE_AT debug_libs 0) - list(FILTER debug_libs EXCLUDE REGEX [[^\\[$]<\\[$]>:]]) - list(TRANSFORM debug_libs REPLACE [[^\\[$]<\\[$]:(.*)>$]] "\\1") - - set(merged_libs "") - foreach(release_lib debug_lib IN ZIP_LISTS release_libs debug_libs) - if(release_lib STREQUAL debug_lib) - list(APPEND merged_libs "${release_lib}") - else() - if(release_lib) - list(APPEND merged_libs "\\\$<\\\$>:${release_lib}>") - endif() - if(debug_lib) - list(APPEND merged_libs "\\\$<\\\$:${debug_lib}>") - endif() - endif() - endforeach() - set("${out_var}" "${merged_libs}" PARENT_SCOPE) -endfunction() diff --git a/scripts/install-gcs-emu.sh b/scripts/install-gcs-emu.sh index 0cfc74adcdb7..d9391555be43 100755 --- a/scripts/install-gcs-emu.sh +++ b/scripts/install-gcs-emu.sh @@ -31,7 +31,7 @@ die() { } install_gcs(){ - git clone --branch v0.36.0 --depth 1 https://github.com/googleapis/storage-testbench.git /tmp/storage-testbench + git clone --branch v0.45.0 --depth 1 https://github.com/googleapis/storage-testbench.git /tmp/storage-testbench # Create a virtual environment and keep it active python3 -m venv /tmp/storage-testbench-venv source /tmp/storage-testbench-venv/bin/activate diff --git a/scripts/install-run-gcs-emu.ps1 b/scripts/install-run-gcs-emu.ps1 index 59e9a85e7dde..ae47f80003a8 100644 --- a/scripts/install-run-gcs-emu.ps1 +++ b/scripts/install-run-gcs-emu.ps1 @@ -23,7 +23,7 @@ Param( $ErrorActionPreference = "Stop" -$version = "v0.39.0" +$version = "v0.45.0" $testbenchPath = "$env:TEMP\storage-testbench-$version" $venvPath = "$env:TEMP\storage-testbench-venv" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bf7c60f6b221..57a10e792306 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,62 +28,6 @@ find_package(Catch2 REQUIRED) -# Arrow integration test config and dependencies -# Override from environment if not set in cache -if (NOT DEFINED $CACHE{TILEDB_ARROW_TESTS}) - if ($ENV{TILEDB_ARROW_TESTS}) - message(STATUS "Enabling Apache Arrow integration test") - # Technically this would be an "option", but we want conditional override from ENV - set(TILEDB_ARROW_TESTS ON CACHE BOOL "Enable Arrow tests (requires Python with pyarrow and numpy)" FORCE) - mark_as_advanced(TILEDB_ARROW_TESTS) - endif() -endif() - -if (${TILEDB_ARROW_TESTS}) - # Reworked FindPython was introducted in 3.12 with features used below - if (CMAKE_VERSION VERSION_LESS "3.12") - message(FATAL_ERROR "CMake >= 3.12 is required for TileDB Arrow Tests. (found ${CMAKE_VERSION})") - endif() - # Tell CMake to check the Python registry entry last on Windows - set(Python_FIND_REGISTRY "LAST") - # Tell CMake to prefer Python from the PATH - set(Python_FIND_STRATEGY "LOCATION") - find_package(Python COMPONENTS Interpreter Development REQUIRED) - find_package(pybind11) - - message(STATUS "Configuring Apache Arrow integration test with Python ${Python_VERSION} (${Python_EXECUTABLE})") - - # If we can't find the pybind11 cmake config (not available in pypi yet) - # try to find with the current executable. - if (NOT ${pybind11_FOUND}) - # Get the include arguments from the python executable (has "-I" compiler option) - execute_process(COMMAND ${Python_EXECUTABLE} -m pybind11 --includes - OUTPUT_VARIABLE CMD_PYBIND11_INCLUDE - RESULT_VARIABLE CMD_PYBIND11_RESULT - OUTPUT_STRIP_TRAILING_WHITESPACE) - if (${CMD_PYBIND11_RESULT}) - message(FATAL_ERROR "Unable to find pybind11 via cmake or 'python3 -m pybind11 --includes'") - endif() - - # Convert args to list - separate_arguments(CMD_PARSED_INCLUDES NATIVE_COMMAND ${CMD_PYBIND11_INCLUDE}) - # Remove the "-I" from each include - foreach(INCL_PATH IN LISTS CMD_PARSED_INCLUDES) - string(REPLACE "-I" "" INCL_PATH ${INCL_PATH}) - list(APPEND PYBIND11_INCLUDE_DIRECTORIES ${INCL_PATH}) - endforeach() - - file(TO_CMAKE_PATH "${Python_SITELIB}" SAFE_Python_SITELIB) - set(pybind11_FOUND TRUE CACHE BOOL "pybind11 include path found") - add_library(pybind11::embed INTERFACE IMPORTED) - target_include_directories(pybind11::embed INTERFACE ${PYBIND11_INCLUDE_DIRECTORIES}) - target_link_libraries(pybind11::embed INTERFACE Python::Python) - target_compile_definitions(pybind11::embed INTERFACE -DTILEDB_PYTHON_SITELIB_PATH="${SAFE_Python_SITELIB}") - endif() - file(TO_CMAKE_PATH ${CMAKE_CURRENT_BINARY_DIR} SAFE_CURRENT_BINARY_DIR) - target_compile_definitions(pybind11::embed INTERFACE -DTILEDB_PYTHON_UNIT_PATH="${SAFE_CURRENT_BINARY_DIR}") -endif() - # Include TileDB core header directories set(TILEDB_CORE_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/..") # Include the C API directory so that the C++ 'tiledb' file can directly @@ -167,7 +111,6 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-compression-dd.cc src/unit-compression-delta.cc src/unit-compression-rle.cc - src/unit-crypto.cc src/unit-ctx.cc src/unit-current-domain-rest.cc src/unit-dense-reader.cc @@ -260,17 +203,15 @@ if (TILEDB_SERIALIZATION) ) endif() -if (TILEDB_ARROW_TESTS) - list(APPEND TILEDB_UNIT_TEST_SOURCES - src/unit-arrow.cc - ${CMAKE_SOURCE_DIR}/tiledb/sm/cpp_api/arrow_io_impl.h - ) -endif() - if (TILEDB_VERBOSE) add_definitions(-DTILEDB_VERBOSE) endif() +# We want tests to continue as normal even as the API is changing, +# so don't warn for deprecations, since they'll be escalated to errors. +if (NOT MSVC) + add_compile_options(-Wno-deprecated-declarations) +endif() # unit test executable add_executable( @@ -280,16 +221,8 @@ add_executable( "src/unit.cc" ) -add_dependencies(tiledb_unit tiledb_test_support_lib) - target_compile_options(tiledb_unit PRIVATE "$<$:/utf-8>") -# We want tests to continue as normal even as the API is changing, -# so don't warn for deprecations, since they'll be escalated to errors. -if (NOT MSVC) - target_compile_options(tiledb_unit PRIVATE -Wno-deprecated-declarations) -endif() - target_include_directories( tiledb_unit BEFORE PRIVATE ${TILEDB_CORE_INCLUDE_DIR} @@ -330,13 +263,6 @@ if (TILEDB_SERIALIZATION) target_compile_definitions(tiledb_unit PRIVATE -DTILEDB_SERIALIZATION) endif() -if (TILEDB_ARROW_TESTS) - target_link_libraries(tiledb_unit PRIVATE pybind11::embed) - - # install the python helper next to the executable for import - configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/unit_arrow.py" "${CMAKE_CURRENT_BINARY_DIR}" COPYONLY) -endif() - if (TILEDB_WEBP) target_compile_definitions(tiledb_unit PRIVATE -DTILEDB_WEBP) find_package(ZLIB) # We need PNG to use our Zlib so that static link works correctly if applicable @@ -364,6 +290,50 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU ) endif() +if (TILEDB_ARROW_TESTS) + # Tell CMake to check the Python registry entry last on Windows + set(Python_FIND_REGISTRY "LAST") + # Tell CMake to prefer Python from the PATH + set(Python_FIND_STRATEGY "LOCATION") + find_package(Python COMPONENTS Interpreter Development REQUIRED) + find_package(pybind11 REQUIRED) + message(STATUS "Configuring Apache Arrow integration test with Python ${Python_VERSION} (${Python_EXECUTABLE})") + + add_executable( + unit_arrow EXCLUDE_FROM_ALL + $ + src/unit-arrow.cc + ${CMAKE_SOURCE_DIR}/tiledb/sm/cpp_api/arrow_io_impl.h + ) + + target_link_libraries(unit_arrow + PUBLIC + TILEDB_CORE_OBJECTS_ILIB + TILEDB_CORE_OBJECTS + Catch2::Catch2WithMain + pybind11::embed + tiledb_test_support_lib + configuration_definitions + ) + + file(TO_CMAKE_PATH ${CMAKE_CURRENT_BINARY_DIR} SAFE_CURRENT_BINARY_DIR) + target_compile_definitions(unit_arrow PRIVATE -DTILEDB_PYTHON_UNIT_PATH="${SAFE_CURRENT_BINARY_DIR}") + # install the python helper next to the executable for import + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/unit_arrow.py" "${CMAKE_CURRENT_BINARY_DIR}" COPYONLY) + + target_include_directories( + unit_arrow BEFORE PRIVATE + ${TILEDB_CORE_INCLUDE_DIR} + ${TILEDB_EXPORT_HEADER_DIR} + ) + + add_test( + NAME "unit_arrow" + COMMAND unit_arrow --durations=yes + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) +endif() + # Only produce timing tests for UNIX based systems (faketime constraint) find_library( LIBFAKETIME diff --git a/test/performance/msys_handle_leakage/unit-cppapi-consolidation-with-timestamps.cc b/test/performance/msys_handle_leakage/unit-cppapi-consolidation-with-timestamps.cc index 880a97500382..51f9c6cc8466 100644 --- a/test/performance/msys_handle_leakage/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/performance/msys_handle_leakage/unit-cppapi-consolidation-with-timestamps.cc @@ -70,13 +70,11 @@ struct ConsolidationWithTimestampsFx { // Functions. void set_legacy(); void create_sparse_array(bool allows_dups = false); - void create_sparse_array_v11(); void write_sparse( std::vector a1, std::vector dim1, std::vector dim2, uint64_t timestamp); - void write_sparse_v11(uint64_t timestamp); void consolidate_sparse(bool vacuum = false); void consolidate_sparse( uint64_t start_time, uint64_t end_time, bool vacuum = false); @@ -160,18 +158,6 @@ void ConsolidationWithTimestampsFx::create_sparse_array(bool allows_dups) { Array::create(SPARSE_ARRAY_NAME, schema); } -void ConsolidationWithTimestampsFx::create_sparse_array_v11() { - // Get the v11 sparse array. - std::string v11_arrays_dir = - std::string(TILEDB_TEST_INPUTS_DIR) + "/arrays/sparse_array_v11"; - REQUIRE( - tiledb_vfs_copy_dir( - ctx_.ptr().get(), - vfs_.ptr().get(), - v11_arrays_dir.c_str(), - SPARSE_ARRAY_NAME) == TILEDB_OK); -} - void ConsolidationWithTimestampsFx::write_sparse( std::vector a1, std::vector dim1, @@ -195,37 +181,6 @@ void ConsolidationWithTimestampsFx::write_sparse( array.close(); } -void ConsolidationWithTimestampsFx::write_sparse_v11(uint64_t timestamp) { - // Prepare cell buffers. - std::vector buffer_a1{0, 1, 2, 3}; - std::vector buffer_a2{0, 1, 3, 6}; - std::string buffer_var_a2("abbcccdddd"); - std::vector buffer_a3{0.1f, 0.2f, 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f}; - std::vector buffer_coords_dim1{1, 1, 1, 2}; - std::vector buffer_coords_dim2{1, 2, 4, 3}; - - // Open array. - Array array(ctx_, SPARSE_ARRAY_NAME, TILEDB_WRITE, timestamp); - - // Create query. - Query query(ctx_, array, TILEDB_WRITE); - query.set_layout(TILEDB_GLOBAL_ORDER); - query.set_data_buffer("a1", buffer_a1); - query.set_data_buffer( - "a2", (void*)buffer_var_a2.c_str(), buffer_var_a2.size()); - query.set_offsets_buffer("a2", buffer_a2); - query.set_data_buffer("a3", buffer_a3); - query.set_data_buffer("d1", buffer_coords_dim1); - query.set_data_buffer("d2", buffer_coords_dim2); - - // Submit/finalize the query. - query.submit(); - query.finalize(); - - // Close array. - array.close(); -} - void ConsolidationWithTimestampsFx::consolidate_sparse(bool vacuum) { auto config = ctx_.config(); Array::consolidate(ctx_, SPARSE_ARRAY_NAME, &config); diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index e7ecf827fe91..370d24abfd4f 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -51,6 +51,10 @@ if (TILEDB_CPP_API) list(APPEND SOURCES targets/sc-35424.cc) list(APPEND SOURCES targets/sc-36372.cc) list(APPEND SOURCES targets/sc-38300.cc) + list(APPEND SOURCES targets/sc-52975.cc) + list(APPEND SOURCES targets/sc-53334.cc) + list(APPEND SOURCES targets/sc-53791.cc) + list(APPEND SOURCES targets/sc-53970.cc) endif() add_executable(tiledb_regression diff --git a/test/regression/targets/sc-52975.cc b/test/regression/targets/sc-52975.cc new file mode 100644 index 000000000000..698bff2f12a0 --- /dev/null +++ b/test/regression/targets/sc-52975.cc @@ -0,0 +1,354 @@ +/** + * @file sc-52975.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include +#include +#include + +#include +#include + +#include + +using namespace tiledb; + +static void create_array(const std::string& array_uri); +static void write_array(const std::string& array_uri, int8_t min, int8_t max); +static bool read_array(const std::string& array_uri, int8_t min, int8_t max); + +std::vector get_attr_data(); +std::vector get_attr_validity(); +std::vector> get_subarrays(); + +TEST_CASE( + "SC-52975: Subarray values lead to incorrect read.", + "[reproducer][bug][sc52975][!shouldfail]") { + std::string array_uri = "test_reproducer"; + + std::size_t successes = 0; + std::size_t failures = 0; + + auto subarrays = get_subarrays(); + std::size_t num_cases = subarrays.size(); + + for (auto subarray : subarrays) { + create_array(array_uri); + write_array(array_uri, subarray.first, subarray.second); + if (read_array(array_uri, subarray.first, subarray.second)) { + successes += 1; + } else { + failures += 1; + } + } + + REQUIRE(((successes == num_cases) && (failures == 0))); +} + +void create_array(const std::string& array_uri) { + Context ctx; + + auto obj = Object::object(ctx, array_uri); + if (obj.type() != Object::Type::Invalid) { + Object::remove(ctx, array_uri); + } + + auto dim = Dimension::create(ctx, "dim", {{-91, 90}}, 1); + + Domain dom(ctx); + dom.add_dimension(dim); + + uint64_t fill_value = 15110407019492443053llu; + + auto attr = Attribute::create(ctx, "attr"); + attr.set_nullable(true); + attr.set_fill_value(&fill_value, sizeof(uint64_t), 0); + // Maybe: FilterList [BitShuffle, ByteShuffle] + + ArraySchema schema(ctx, TILEDB_DENSE); + schema.set_capacity(340418807209297907) + .set_allows_dups(false) + .set_cell_order(TILEDB_ROW_MAJOR) + .set_tile_order(TILEDB_ROW_MAJOR) + .set_domain(dom) + .add_attribute(attr); + + Array::create(array_uri, schema); +} + +void write_array(const std::string& array_uri, int8_t min, int8_t max) { + Context ctx; + Array array(ctx, array_uri, TILEDB_WRITE); + Query query(ctx, array, TILEDB_WRITE); + + Subarray subarray(ctx, array); + subarray.add_range(0, min, max); + + auto attr_data = get_attr_data(); + auto attr_validity = get_attr_validity(); + + attr_data.resize(max - min + 1); + attr_validity.resize(max - min + 1); + + query.set_layout(TILEDB_ROW_MAJOR) + .set_subarray(subarray) + .set_data_buffer("attr", attr_data) + .set_validity_buffer("attr", attr_validity); + + REQUIRE(query.submit() == Query::Status::COMPLETE); + + array.close(); +} + +bool read_array(const std::string& array_uri, int8_t min, int8_t max) { + Context ctx; + Array array(ctx, array_uri, TILEDB_READ); + + Subarray subarray(ctx, array); + subarray.add_range(0, min, max); + + std::vector dim(1024); + std::vector data(1024); + std::vector validity(1024); + + Query query(ctx, array, TILEDB_READ); + query.set_layout(TILEDB_ROW_MAJOR) + .set_subarray(subarray) + .set_data_buffer("dim", dim) + .set_data_buffer("attr", data) + .set_validity_buffer("attr", validity); + REQUIRE(query.submit() == Query::Status::COMPLETE); + + dim.resize(max - min + 1); + data.resize(max - min + 1); + validity.resize(max - min + 1); + + uint64_t fill_value = 15110407019492443053llu; + for (auto d : data) { + if (d == fill_value) { + return false; + } + } + for (auto v : validity) { + if (v == 0) { + return false; + } + } + + return true; +} + +std::vector get_attr_data() { + std::vector data(1024); + std::iota(data.begin(), data.end(), 0); + return data; +} + +std::vector get_attr_validity() { + std::vector validity(1024, 1); + return validity; +} + +// Any value past 37 will cause issues here as 37 - (-91) = 128. Some of our +// computations will interpret 128 as an int8 when they shouldn't and use a +// negative number. This will require a full revision of the dense code to fix +// all erroneous int8 conversions, which is a lot of code to correct. +std::vector> get_subarrays() { + return { + {-77, -77}, {-72, -72}, {-71, -60}, {-70, -31}, {-67, -34}, {-67, -57}, + {-66, -63}, {-58, -43}, {-58, -48}, {-57, -56}, {-56, -48}, {-53, 34}, + {-52, -52}, {-50, -39}, {-47, 15}, {-44, -21}, {-43, -32}, {-42, -15}, + {-42, -34}, {-41, -33}, {-40, -20}, {-38, -3}, {-37, -23}, {-36, -28}, + {-36, -34}, {-34, -33}, {-29, -25}, {-29, -26}, {-29, -27}, {-29, 12}, + {-28, -25}, {-28, 12}, {-26, -15}, {-25, 1}, {-25, 26}, {-23, 6}, + {-22, -20}, {-22, 29}, {-22, 33}, {-22, 35}, {-22, 36}, {-22, 37}, + {-22, 37}, {-22, 44}, {-22, 58}, {-18, -15}, {-17, -17}, {-17, 19}, + {-17, 29}, {-17, 34}, {-17, 36}, {-17, 37}, {-17, 38}, {-16, -1}, + {-15, 27}, {-14, -10}, {-14, -5}, {-14, -7}, {-14, -8}, {-14, 12}, + {-11, -10}, {-11, 21}, {-11, 29}, {-11, 32}, {-11, 35}, {-11, 36}, + {-11, 37}, {-11, 37}, {-11, 38}, {-11, 43}, {-11, 6}, {-10, 10}, + {-10, 7}, {-8, -2}, {-8, -2}, {-8, 23}, {-8, 23}, {-8, 35}, + {-8, 36}, {-8, 37}, {-8, 37}, {-8, 38}, {-8, 4}, {-8, 41}, + {-8, 47}, {-7, -4}, {-7, 0}, {-5, 14}, {-5, 24}, {-5, 31}, + {-5, 34}, {-5, 36}, {-5, 37}, {-5, 37}, {-5, 49}, {-4, -4}, + {-4, 10}, {-4, 14}, {-4, 20}, {-4, 31}, {-4, 34}, {-4, 36}, + {-4, 36}, {-4, 37}, {-4, 37}, {-4, 37}, {-4, 38}, {-4, 39}, + {-4, 39}, {-4, 41}, {-4, 43}, {-4, 51}, {-4, 68}, {-4, 9}, + {-2, -2}, {-2, 1}, {-2, 10}, {-2, 16}, {-2, 20}, {-2, 22}, + {-2, 22}, {-2, 3}, {-2, 30}, {-2, 31}, {-2, 34}, {-2, 34}, + {-2, 36}, {-2, 36}, {-2, 36}, {-2, 37}, {-2, 37}, {-2, 37}, + {-2, 37}, {-2, 37}, {-2, 38}, {-2, 39}, {-2, 40}, {-2, 40}, + {-2, 41}, {-2, 45}, {-2, 45}, {-1, 18}, {-1, 24}, {-1, 27}, + {-1, 27}, {-1, 28}, {-1, 31}, {-1, 33}, {-1, 34}, {-1, 34}, + {-1, 35}, {-1, 36}, {-1, 36}, {-1, 36}, {-1, 37}, {-1, 37}, + {-1, 37}, {-1, 37}, {-1, 37}, {-1, 38}, {-1, 41}, {-1, 49}, + {-1, 5}, {-1, 55}, {0, 0}, {0, 11}, {0, 14}, {0, 18}, + {0, 19}, {0, 19}, {0, 20}, {0, 21}, {0, 22}, {0, 23}, + {0, 24}, {0, 25}, {0, 25}, {0, 26}, {0, 26}, {0, 27}, + {0, 27}, {0, 28}, {0, 28}, {0, 28}, {0, 29}, {0, 29}, + {0, 31}, {0, 31}, {0, 32}, {0, 32}, {0, 33}, {0, 33}, + {0, 33}, {0, 33}, {0, 34}, {0, 34}, {0, 34}, {0, 34}, + {0, 35}, {0, 35}, {0, 35}, {0, 35}, {0, 35}, {0, 35}, + {0, 35}, {0, 36}, {0, 36}, {0, 36}, {0, 36}, {0, 36}, + {0, 36}, {0, 36}, {0, 36}, {0, 36}, {0, 36}, {0, 36}, + {0, 36}, {0, 36}, {0, 37}, {0, 37}, {0, 37}, {0, 37}, + {0, 37}, {0, 37}, {0, 37}, {0, 37}, {0, 37}, {0, 37}, + {0, 37}, {0, 37}, {0, 37}, {0, 37}, {0, 37}, {0, 37}, + {0, 37}, {0, 37}, {0, 37}, {0, 37}, {0, 38}, {0, 38}, + {0, 38}, {0, 38}, {0, 38}, {0, 38}, {0, 39}, {0, 39}, + {0, 39}, {0, 39}, {0, 39}, {0, 4}, {0, 40}, {0, 41}, + {0, 41}, {0, 41}, {0, 42}, {0, 42}, {0, 46}, {0, 47}, + {0, 5}, {0, 50}, {0, 52}, {0, 53}, {0, 55}, {0, 56}, + {0, 62}, {0, 75}, {0, 8}, {1, 1}, {1, 13}, {1, 13}, + {1, 13}, {1, 15}, {1, 16}, {1, 18}, {1, 19}, {1, 19}, + {1, 19}, {1, 2}, {1, 20}, {1, 20}, {1, 21}, {1, 21}, + {1, 21}, {1, 21}, {1, 21}, {1, 22}, {1, 23}, {1, 25}, + {1, 26}, {1, 27}, {1, 29}, {1, 29}, {1, 3}, {1, 30}, + {1, 30}, {1, 31}, {1, 31}, {1, 31}, {1, 32}, {1, 34}, + {1, 34}, {1, 34}, {1, 34}, {1, 34}, {1, 35}, {1, 35}, + {1, 35}, {1, 35}, {1, 36}, {1, 36}, {1, 36}, {1, 36}, + {1, 36}, {1, 36}, {1, 36}, {1, 36}, {1, 36}, {1, 37}, + {1, 37}, {1, 37}, {1, 37}, {1, 37}, {1, 37}, {1, 37}, + {1, 37}, {1, 37}, {1, 37}, {1, 37}, {1, 37}, {1, 38}, + {1, 38}, {1, 38}, {1, 38}, {1, 38}, {1, 38}, {1, 38}, + {1, 39}, {1, 39}, {1, 39}, {1, 39}, {1, 41}, {1, 41}, + {1, 41}, {1, 41}, {1, 41}, {1, 42}, {1, 45}, {1, 46}, + {1, 54}, {1, 59}, {1, 6}, {1, 75}, {1, 9}, {1, 9}, + {2, 17}, {2, 19}, {2, 19}, {2, 2}, {2, 21}, {2, 22}, + {2, 23}, {2, 27}, {2, 28}, {2, 28}, {2, 28}, {2, 31}, + {2, 33}, {2, 33}, {2, 33}, {2, 34}, {2, 35}, {2, 35}, + {2, 35}, {2, 36}, {2, 36}, {2, 36}, {2, 36}, {2, 36}, + {2, 36}, {2, 37}, {2, 37}, {2, 37}, {2, 37}, {2, 37}, + {2, 37}, {2, 37}, {2, 37}, {2, 37}, {2, 37}, {2, 38}, + {2, 38}, {2, 38}, {2, 40}, {2, 40}, {2, 41}, {2, 43}, + {2, 45}, {2, 54}, {2, 72}, {2, 79}, {3, 10}, {3, 20}, + {3, 23}, {3, 28}, {3, 29}, {3, 33}, {3, 34}, {3, 35}, + {3, 36}, {3, 36}, {3, 36}, {3, 37}, {3, 37}, {3, 37}, + {3, 37}, {3, 38}, {3, 38}, {3, 38}, {3, 41}, {3, 43}, + {3, 54}, {3, 9}, {4, 14}, {4, 16}, {4, 22}, {4, 23}, + {4, 23}, {4, 23}, {4, 23}, {4, 32}, {4, 33}, {4, 33}, + {4, 33}, {4, 35}, {4, 36}, {4, 36}, {4, 36}, {4, 36}, + {4, 37}, {4, 37}, {4, 37}, {4, 37}, {4, 37}, {4, 38}, + {4, 38}, {4, 38}, {4, 41}, {4, 42}, {4, 42}, {4, 43}, + {4, 5}, {4, 8}, {4, 9}, {5, 10}, {5, 12}, {5, 13}, + {5, 21}, {5, 22}, {5, 24}, {5, 24}, {5, 26}, {5, 30}, + {5, 31}, {5, 31}, {5, 32}, {5, 34}, {5, 34}, {5, 35}, + {5, 35}, {5, 35}, {5, 36}, {5, 36}, {5, 36}, {5, 36}, + {5, 36}, {5, 37}, {5, 37}, {5, 37}, {5, 37}, {5, 37}, + {5, 37}, {5, 37}, {5, 37}, {5, 37}, {5, 38}, {5, 38}, + {5, 39}, {5, 39}, {5, 44}, {5, 44}, {5, 44}, {5, 45}, + {5, 47}, {5, 57}, {5, 58}, {6, 11}, {6, 12}, {6, 15}, + {6, 15}, {6, 15}, {6, 16}, {6, 18}, {6, 22}, {6, 22}, + {6, 22}, {6, 24}, {6, 26}, {6, 29}, {6, 29}, {6, 30}, + {6, 31}, {6, 31}, {6, 32}, {6, 32}, {6, 34}, {6, 34}, + {6, 35}, {6, 35}, {6, 35}, {6, 36}, {6, 36}, {6, 36}, + {6, 36}, {6, 37}, {6, 37}, {6, 37}, {6, 37}, {6, 37}, + {6, 37}, {6, 37}, {6, 38}, {6, 38}, {6, 39}, {6, 41}, + {6, 47}, {6, 53}, {6, 7}, {6, 9}, {6, 9}, {7, 12}, + {7, 21}, {7, 22}, {7, 22}, {7, 23}, {7, 30}, {7, 31}, + {7, 31}, {7, 32}, {7, 34}, {7, 34}, {7, 35}, {7, 36}, + {7, 36}, {7, 36}, {7, 36}, {7, 37}, {7, 37}, {7, 37}, + {7, 37}, {7, 37}, {7, 37}, {7, 37}, {7, 38}, {7, 39}, + {7, 39}, {7, 43}, {7, 45}, {7, 55}, {7, 57}, {7, 67}, + {8, 10}, {8, 12}, {8, 13}, {8, 16}, {8, 19}, {8, 23}, + {8, 24}, {8, 24}, {8, 26}, {8, 27}, {8, 28}, {8, 32}, + {8, 32}, {8, 33}, {8, 33}, {8, 34}, {8, 35}, {8, 35}, + {8, 35}, {8, 36}, {8, 36}, {8, 36}, {8, 36}, {8, 36}, + {8, 37}, {8, 37}, {8, 37}, {8, 37}, {8, 37}, {8, 37}, + {8, 37}, {8, 37}, {8, 37}, {8, 38}, {8, 39}, {8, 40}, + {8, 40}, {8, 41}, {8, 46}, {8, 47}, {8, 49}, {8, 59}, + {8, 9}, {9, 10}, {9, 23}, {9, 27}, {9, 28}, {9, 30}, + {9, 30}, {9, 32}, {9, 34}, {9, 35}, {9, 35}, {9, 36}, + {9, 36}, {9, 36}, {9, 37}, {9, 37}, {9, 37}, {9, 37}, + {9, 37}, {9, 38}, {9, 39}, {9, 41}, {9, 46}, {9, 52}, + {9, 65}, {10, 16}, {10, 19}, {10, 24}, {10, 24}, {10, 29}, + {10, 31}, {10, 32}, {10, 34}, {10, 35}, {10, 36}, {10, 36}, + {10, 36}, {10, 36}, {10, 37}, {10, 37}, {10, 37}, {10, 37}, + {10, 37}, {10, 37}, {10, 38}, {10, 38}, {10, 38}, {10, 39}, + {10, 39}, {10, 40}, {10, 43}, {10, 48}, {10, 49}, {10, 62}, + {10, 67}, {10, 69}, {11, 28}, {11, 31}, {11, 33}, {11, 35}, + {11, 36}, {11, 36}, {11, 36}, {11, 37}, {11, 37}, {11, 37}, + {11, 37}, {11, 38}, {11, 38}, {11, 39}, {11, 40}, {11, 41}, + {11, 43}, {11, 46}, {11, 49}, {11, 51}, {11, 62}, {12, 27}, + {12, 33}, {12, 35}, {12, 36}, {12, 36}, {12, 37}, {12, 37}, + {12, 37}, {12, 38}, {12, 39}, {12, 39}, {12, 43}, {12, 44}, + {12, 54}, {12, 74}, {13, 13}, {13, 13}, {13, 16}, {13, 17}, + {13, 26}, {13, 28}, {13, 29}, {13, 29}, {13, 31}, {13, 33}, + {13, 33}, {13, 35}, {13, 36}, {13, 36}, {13, 36}, {13, 36}, + {13, 37}, {13, 37}, {13, 37}, {13, 37}, {13, 37}, {13, 38}, + {13, 38}, {13, 38}, {13, 39}, {13, 39}, {13, 39}, {13, 41}, + {13, 42}, {13, 45}, {13, 48}, {13, 50}, {13, 60}, {14, 25}, + {14, 26}, {14, 28}, {14, 28}, {14, 32}, {14, 35}, {14, 35}, + {14, 35}, {14, 36}, {14, 36}, {14, 36}, {14, 37}, {14, 37}, + {14, 37}, {14, 37}, {14, 37}, {14, 37}, {14, 38}, {14, 39}, + {14, 39}, {14, 42}, {14, 42}, {14, 63}, {14, 70}, {15, 24}, + {15, 24}, {15, 26}, {15, 29}, {15, 30}, {15, 32}, {15, 35}, + {15, 36}, {15, 36}, {15, 37}, {15, 37}, {15, 37}, {15, 38}, + {15, 40}, {15, 43}, {16, 28}, {16, 31}, {16, 33}, {16, 34}, + {16, 35}, {16, 36}, {16, 36}, {16, 37}, {16, 37}, {16, 37}, + {16, 37}, {16, 38}, {16, 40}, {16, 40}, {16, 45}, {16, 55}, + {17, 21}, {17, 23}, {17, 30}, {17, 33}, {17, 35}, {17, 36}, + {17, 36}, {17, 37}, {17, 37}, {17, 37}, {17, 38}, {17, 38}, + {17, 40}, {17, 42}, {17, 45}, {17, 50}, {17, 54}, {18, 32}, + {18, 34}, {18, 36}, {18, 37}, {18, 38}, {18, 39}, {18, 46}, + {18, 75}, {19, 20}, {19, 21}, {19, 34}, {20, 29}, {20, 30}, + {20, 30}, {20, 32}, {20, 33}, {20, 34}, {20, 35}, {20, 35}, + {20, 36}, {20, 36}, {20, 36}, {20, 36}, {20, 37}, {20, 37}, + {20, 37}, {20, 37}, {20, 37}, {20, 37}, {20, 38}, {20, 38}, + {20, 38}, {20, 39}, {20, 40}, {20, 40}, {20, 45}, {20, 47}, + {20, 57}, {20, 70}, {21, 29}, {21, 33}, {21, 35}, {21, 36}, + {21, 37}, {21, 37}, {21, 54}, {22, 22}, {22, 25}, {23, 33}, + {23, 36}, {23, 37}, {23, 38}, {23, 43}, {24, 25}, {24, 25}, + {24, 36}, {24, 37}, {24, 38}, {24, 40}, {24, 43}, {24, 49}, + {25, 31}, {25, 32}, {25, 35}, {25, 36}, {25, 36}, {25, 36}, + {25, 37}, {25, 37}, {25, 37}, {25, 37}, {25, 38}, {25, 38}, + {25, 38}, {25, 39}, {25, 40}, {25, 42}, {25, 47}, {25, 51}, + {25, 55}, {26, 27}, {26, 34}, {26, 36}, {26, 37}, {26, 38}, + {26, 42}, {26, 59}, {27, 32}, {27, 32}, {27, 33}, {27, 35}, + {27, 35}, {27, 36}, {27, 36}, {27, 37}, {27, 37}, {27, 37}, + {27, 37}, {27, 38}, {27, 48}, {27, 49}, {27, 69}, {28, 30}, + {28, 33}, {28, 34}, {28, 36}, {28, 36}, {28, 37}, {28, 37}, + {28, 37}, {28, 38}, {28, 40}, {28, 49}, {28, 53}, {28, 71}, + {28, 79}, {30, 36}, {31, 34}, {31, 36}, {31, 37}, {31, 38}, + {31, 46}, {31, 61}, {32, 35}, {32, 36}, {32, 37}, {32, 37}, + {32, 38}, {32, 44}, {34, 36}, {34, 37}, {34, 38}, {34, 43}, + {34, 52}, {34, 71}, {35, 36}, {35, 36}, {35, 36}, {35, 37}, + {35, 37}, {35, 37}, {35, 38}, {35, 39}, {35, 42}, {35, 44}, + {35, 54}, {37, 42}, {37, 65}, {38, 46}, {39, 52}, {40, 40}, + {41, 44}, {42, 43}, {42, 45}, {43, 44}, {44, 48}, {44, 62}, + {46, 76}, {51, 55}, {51, 74}, {52, 56}, {52, 60}, {52, 63}, + {54, 55}, {55, 58}, {55, 63}, {58, 60}, {61, 63}, {61, 63}, + {61, 75}, {62, 63}, {63, 63}, {64, 65}, {66, 66}, {66, 69}, + {68, 68}, {68, 68}, {68, 68}, {69, 69}, {69, 79}, {70, 70}, + {70, 80}, {71, 73}, {73, 75}, {74, 74}, {74, 77}, {74, 78}, + {74, 82}, {75, 76}, {75, 79}, {76, 80}, {77, 78}, {77, 78}, + {77, 81}, {77, 83}, {79, 79}, {79, 81}, {79, 83}, {80, 80}, + {80, 87}, {81, 81}, {81, 81}, {81, 82}, {82, 82}, {83, 83}, + {83, 84}, {84, 86}, {87, 88}, {88, 88}, {88, 88}, {88, 89}, + {89, 89}, {89, 89}, {89, 89}, {89, 89}, {90, 90}}; +} diff --git a/test/regression/targets/sc-53334.cc b/test/regression/targets/sc-53334.cc new file mode 100644 index 000000000000..0bc8b0a17266 --- /dev/null +++ b/test/regression/targets/sc-53334.cc @@ -0,0 +1,194 @@ +/** + * @file sc-53334.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + */ + +#include +#include +#include +#include +#include +#include + +#include + +using namespace tiledb; + +typedef char min_type; + +template +struct AttributeTraits; + +template <> +struct AttributeTraits { + typedef uint8_t value_type; +}; + +template <> +struct AttributeTraits { + typedef char value_type; +}; + +template +struct MyArray { + using AttributeValueType = + typename AttributeTraits::value_type; + + static void create_array(Context& ctx, const char* array_name) { + // The array will be 4x4 with dimensions "rows" and "cols", with domain + // [1,4]. + auto rows = Dimension::create(ctx, "rows", {{1, 4}}, 4); + auto cols = Dimension::create(ctx, "columns", {{1, 4}}, 4); + + // Create domain + Domain domain(ctx); + domain.add_dimension(rows); + domain.add_dimension(cols); + + // Create a single attribute "a" so each (i,j) cell can store a character + Attribute a(ctx, "a", attribute_datatype); + + // Create array schema + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain); + schema.set_cell_order(TILEDB_ROW_MAJOR); + schema.set_tile_order(TILEDB_ROW_MAJOR); + schema.add_attribute(a); + + // Create array + Array::create(array_name, schema); + } + + static void write_array(Context& ctx, const char* array_name) { + // Open array for writing + Array array(ctx, array_name, TILEDB_WRITE); + + // Data + std::vector rows = { + 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4}; + std::vector cols = { + 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4}; + std::vector atts = { + 'a', + 'b', + 'c', + 'd', + 'e', + 'f', + 'g', + 'h', + 'i', + 'j', + 'k', + 'l', + 'm', + 'n', + 'o', + 'p'}; + + Query query(ctx, array); + query.set_data_buffer("rows", rows) + .set_data_buffer("columns", cols) + .set_data_buffer("a", atts); + + query.submit(); + array.close(); + } + + static AttributeValueType query_min(Context& ctx, const char* array_name) { + // note, use C API because the CPP API doesn't seem to have Min yet + Array array(ctx, array_name, TILEDB_READ); + + Query query(ctx, array); + query.set_layout(TILEDB_UNORDERED); + + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate(query, "a"); + default_channel.apply_aggregate("Min", operation); + + std::vector min(1); + query.set_data_buffer("Min", min); + + query.submit(); + query.finalize(); + + return min[0]; + } +}; + +bool array_exists(Context& ctx, const char* uri) { + auto object = tiledb::Object::object(ctx, uri); + return (object.type() == tiledb::Object::Type::Array); +} + +TEST_CASE("SC-53334 min single value UINT8 works", "[bug][sc-53334]") { + using MyArray = MyArray; + + Context ctx; + std::string uri("sc-53334-uint8"); + + if (!array_exists(ctx, uri.c_str())) { + MyArray::create_array(ctx, uri.c_str()); + MyArray::write_array(ctx, uri.c_str()); + } + + const uint8_t min = MyArray::query_min(ctx, uri.c_str()); + REQUIRE(min == 'a'); +} + +TEST_CASE( + "SC-53334 min single value STRING_ASCII does not work", + "[regression][bug][sc-53334][!shouldfail]") { + using MyArray = MyArray; + + Context ctx; + std::string uri("sc-53334-string-ascii"); + + if (!array_exists(ctx, uri.c_str())) { + MyArray::create_array(ctx, uri.c_str()); + MyArray::write_array(ctx, uri.c_str()); + } + + /* + * This throws an exception instead of returning the correct result "a". + * + * "OutputBufferValidator: Aggregate fixed size buffer should be for one + * element" This happens because the Min/Max ops are specialized to do + * std::string as their internal result buffer, but we are looking for a + * single `char` result. + * + * When the bug is fixed, delete `CHECK_THROWS` and `if (false)`. + */ + CHECK_NOTHROW(MyArray::query_min(ctx, uri.c_str())); + if (false) { + const char min = MyArray::query_min(ctx, uri.c_str()); + REQUIRE(min == 'a'); + } +} diff --git a/test/regression/targets/sc-53791.cc b/test/regression/targets/sc-53791.cc new file mode 100644 index 000000000000..95d026a3b687 --- /dev/null +++ b/test/regression/targets/sc-53791.cc @@ -0,0 +1,141 @@ +/** + * @file sc-53791.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include + +using namespace tiledb; + +static void create_array(Context& ctx, const char* array_name) { + auto dx = Dimension::create(ctx, "x", {{1, 100}}, 10); + + // Create domain + Domain domain(ctx); + domain.add_dimension(dx); + + // Create a single attribute "a" so each (i,j) cell can store a character + Attribute a(ctx, "a", TILEDB_UINT64); + a.set_cell_val_num(TILEDB_VAR_NUM); + a.set_nullable(true); + + // Create array schema + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain); + schema.set_cell_order(TILEDB_ROW_MAJOR); + schema.set_tile_order(TILEDB_ROW_MAJOR); + schema.add_attribute(a); + + // Create array + Array::create(array_name, schema); +} + +static void write_array(Context& ctx, const char* array_name) { + // Open array for writing + Array array(ctx, array_name, TILEDB_WRITE); + + // Data + std::vector x = {1}; + std::vector a = {0}; + std::vector a_offsets = {0}; + std::vector a_validity = {1}; + + Query query(ctx, array); + query.set_data_buffer("x", x) + .set_data_buffer("a", a) + .set_offsets_buffer("a", a_offsets) + .set_validity_buffer("a", a_validity); + + query.submit(); + query.finalize(); + array.close(); +} + +static std::pair query_null_count( + Context& ctx, const char* array_name) { + // note, use C API because the CPP API doesn't seem to have Min yet + Array array(ctx, array_name, TILEDB_READ); + + Query query(ctx, array); + query.set_layout(TILEDB_UNORDERED); + + QueryChannel default_channel = QueryExperimental::get_default_channel(query); + default_channel.apply_aggregate("Count", CountOperation()); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate(query, "a"); + default_channel.apply_aggregate("Null Count", operation); + + std::vector count(1); + query.set_data_buffer("Count", count); + + std::vector null_count(1); + query.set_data_buffer("Null Count", null_count); + + /* + * FIXME: this gives an error + * + * Error: Caught std::exception: FragmentMetadata: Trying to + * access tile min metadata that's not present + */ + query.submit(); + query.finalize(); + + return std::make_pair(count[0], null_count[0]); +} + +bool array_exists(Context& ctx, const char* uri); + +TEST_CASE( + "SC-53791 var value TILEDB_UINT64 does not work", + "[regression][bug][sc-53791][!shouldfail]") { + Context ctx; + std::string uri("sc-53791-uint64-var"); + + if (!array_exists(ctx, uri.c_str())) { + create_array(ctx, uri.c_str()); + write_array(ctx, uri.c_str()); + } + + /* + * See the FIXME above. `submit` throws an exception + * due to missing data but we expect it to return normally. + * + * When the bug is fixed, delete `CHECK_NOTHROW` and `if (false)`. + */ + CHECK_NOTHROW(query_null_count(ctx, uri.c_str())); + if (false) { + const auto counts = query_null_count(ctx, uri.c_str()); + REQUIRE(counts.first == 1); + REQUIRE(counts.second == 0); + } +} diff --git a/test/regression/targets/sc-53970.cc b/test/regression/targets/sc-53970.cc new file mode 100644 index 000000000000..67f6eb014918 --- /dev/null +++ b/test/regression/targets/sc-53970.cc @@ -0,0 +1,127 @@ +/** + * @file sc-53970.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + */ +#include + +#include +#include + +#include + +using namespace tiledb; + +static void create_array(const std::string& array_uri); +static void write_array(const std::string& array_uri); + +TEST_CASE( + "Subarray range expansion bug", + "[bug][sc53970][subarray-range-expansion][!shouldfail]") { + std::string array_uri = "test_array_schema_dump"; + + // Test setup + create_array(array_uri); + write_array(array_uri); + + Context ctx; + Array array(ctx, array_uri, TILEDB_READ); + + std::vector dim = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; + std::vector attr = { + -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0}; + + Query query(ctx, array, TILEDB_READ); + + Config cfg; + cfg["sm.var_offsets.bitsize"] = "64"; + cfg["sm.var_offsets.mode"] = "elements"; + cfg["sm.var_offsets.extra_element"] = "true"; + query.set_config(cfg); + + int64_t range1[] = {-1374262780975110845, -1374262780975110845}; + int64_t range2[] = {-6603679540125901718, 0}; + + Subarray subarray(ctx, array); + subarray.add_range(0, range1[0], range1[1]) + .add_range(0, range2[0], range2[1]); + + query.set_layout(TILEDB_UNORDERED) + .set_subarray(subarray) + .set_data_buffer("dim", dim) + .set_data_buffer("attr", attr); + REQUIRE(query.submit() == Query::Status::COMPLETE); + + // The expected result are a single matching cell of (0, 1507468.6) + REQUIRE(dim[0] == 0); + REQUIRE(abs(attr[0] - 1507468.6) < 0.00000005); + + // Check we didn't get any extra results + REQUIRE(dim[1] == -1); + REQUIRE(attr[1] == -1.0); +} + +void create_array(const std::string& array_uri) { + Context ctx; + + auto obj = Object::object(ctx, array_uri); + if (obj.type() != Object::Type::Invalid) { + Object::remove(ctx, array_uri); + } + + auto dim = + Dimension::create(ctx, "dim", {{INT64_MIN, INT64_MAX - 1}}); + + Domain dom(ctx); + dom.add_dimension(dim); + + auto attr = Attribute::create(ctx, "attr"); + + ArraySchema schema(ctx, TILEDB_SPARSE); + + schema.set_order({{TILEDB_COL_MAJOR, TILEDB_COL_MAJOR}}) + .set_domain(dom) + .add_attribute(attr) + .set_capacity(1713) + .set_allows_dups(true); + + Array::create(array_uri, schema); +} + +void write_array(const std::string& array_uri) { + Context ctx; + Array array(ctx, array_uri, TILEDB_WRITE); + Query query(ctx, array, TILEDB_WRITE); + + std::vector dim = {0, -8672700570587565350}; + std::vector attr = {1507468.6f, -0.0f}; + + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("dim", dim) + .set_data_buffer("attr", attr); + REQUIRE(query.submit() == Query::Status::COMPLETE); +} diff --git a/test/src/test-capi-array-write-ordered-attr-fixed.cc b/test/src/test-capi-array-write-ordered-attr-fixed.cc index 264046bcac62..57b9e1b8ea10 100644 --- a/test/src/test-capi-array-write-ordered-attr-fixed.cc +++ b/test/src/test-capi-array-write-ordered-attr-fixed.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB Inc. + * @copyright Copyright (c) 2022-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -37,6 +37,8 @@ using namespace Catch::Matchers; #include #include #include +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" +#include "tiledb/api/c_api/attribute/attribute_api_internal.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/c_api/tiledb.h" @@ -74,7 +76,7 @@ struct FixedOrderedAttributeArrayFixture { // Define the attribute: skip C-API since ordered attributes aren't // exposed in the C-API yet. auto attr = make_shared(HERE(), "a", type, 1, order); - auto st = schema->array_schema_->add_attribute(attr); + auto st = schema->add_attribute(attr); REQUIRE(st.ok()); // Create the array and clean-up. diff --git a/test/src/test-capi-array-write-ordered-attr-var.cc b/test/src/test-capi-array-write-ordered-attr-var.cc index 12aa7554c316..c608c699c90d 100644 --- a/test/src/test-capi-array-write-ordered-attr-var.cc +++ b/test/src/test-capi-array-write-ordered-attr-var.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB Inc. + * @copyright Copyright (c) 2022-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -37,6 +37,8 @@ using namespace Catch::Matchers; #include #include #include +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" +#include "tiledb/api/c_api/attribute/attribute_api_internal.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/c_api/tiledb.h" @@ -75,7 +77,7 @@ struct VarOrderedAttributeArrayFixture { // exposed in the C-API yet. auto attr = make_shared( HERE(), "a", Datatype::STRING_ASCII, constants::var_num, order); - auto st = schema->array_schema_->add_attribute(attr); + auto st = schema->add_attribute(attr); REQUIRE(st.ok()); // Create the array and clean-up. diff --git a/test/src/test-capi-dimension-label.cc b/test/src/test-capi-dimension-label.cc index 08e933a5d038..dd94fc9878b2 100644 --- a/test/src/test-capi-dimension-label.cc +++ b/test/src/test-capi-dimension-label.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,6 +32,7 @@ #include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" #include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_experimental.h" @@ -359,7 +360,7 @@ TEST_CASE_METHOD( // Check array schema and number of dimension labels. require_tiledb_ok(tiledb_array_schema_check(ctx, array_schema)); - auto dim_label_num = array_schema->array_schema_->dim_label_num(); + auto dim_label_num = array_schema->dim_label_num(); REQUIRE(dim_label_num == 1); // Create array @@ -384,10 +385,9 @@ TEST_CASE_METHOD( tiledb_array_schema_t* loaded_dim_label_array_schema{nullptr}; require_tiledb_ok(tiledb_array_schema_load( ctx, dim_label_uri, &loaded_dim_label_array_schema)); - uint64_t loaded_tile_extent{ - loaded_dim_label_array_schema->array_schema_->dimension_ptr(0) - ->tile_extent() - .rvalue_as()}; + uint64_t loaded_tile_extent{loaded_dim_label_array_schema->dimension_ptr(0) + ->tile_extent() + .rvalue_as()}; REQUIRE(tile_extent == loaded_tile_extent); tiledb_array_schema_free(&loaded_dim_label_array_schema); } diff --git a/test/src/test-capi-subarray-labels.cc b/test/src/test-capi-subarray-labels.cc index 6c1359301279..0594f0bdb594 100644 --- a/test/src/test-capi-subarray-labels.cc +++ b/test/src/test-capi-subarray-labels.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,6 +32,7 @@ #include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" #include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_experimental.h" @@ -103,7 +104,7 @@ class SampleLabelledArrayTestFixture : public TemporaryDirectoryFixture { // Check array schema and number of dimension labels. require_tiledb_ok(tiledb_array_schema_check(ctx, array_schema)); - auto dim_label_num = array_schema->array_schema_->dim_label_num(); + auto dim_label_num = array_schema->dim_label_num(); REQUIRE(dim_label_num == 2); // Create array and free array schema. diff --git a/test/src/test-cppapi-current-domain.cc b/test/src/test-cppapi-current-domain.cc index e7f3ae731dc9..fad73f8acd2b 100644 --- a/test/src/test-cppapi-current-domain.cc +++ b/test/src/test-cppapi-current-domain.cc @@ -97,6 +97,13 @@ TEST_CASE_METHOD( range = rect.range(1); CHECK(range[0] == 30); CHECK(range[1] == 40); + + // Check range dtype + CHECK(ndrect.range_dtype(0) == TILEDB_INT32); + CHECK(ndrect.range_dtype("x") == TILEDB_INT32); + + // Check ndim api + CHECK(ndrect.dim_num() == 2); } TEST_CASE_METHOD( diff --git a/test/src/unit-arrow.cc b/test/src/unit-arrow.cc index 48a94f608edf..19fafe38f31a 100644 --- a/test/src/unit-arrow.cc +++ b/test/src/unit-arrow.cc @@ -46,7 +46,7 @@ namespace py = pybind11; namespace { struct CPPArrayFx { public: - CPPArrayFx(std::string uri, const uint64_t col_size) + CPPArrayFx(std::string uri, const uint64_t col_size, const uint8_t offset) : vfs(ctx) , uri(uri) { if (vfs.is_dir(uri)) @@ -59,41 +59,47 @@ struct CPPArrayFx { domain.add_dimensions(d1); std::vector attrs; - attrs.insert( - attrs.end(), - {Attribute::create(ctx, "int8"), - Attribute::create(ctx, "int16"), - Attribute::create(ctx, "int32"), - Attribute::create(ctx, "int64"), - - Attribute::create(ctx, "uint8"), - Attribute::create(ctx, "uint16"), - Attribute::create(ctx, "uint32"), - Attribute::create(ctx, "uint64"), - - Attribute::create(ctx, "float32"), - Attribute::create(ctx, "float64")}); - - // must be constructed manually to get TILEDB_STRING_UTF8 type - { - auto str_attr = Attribute(ctx, "utf_string1", TILEDB_STRING_UTF8); + if (offset == 64) { + auto str_attr = Attribute(ctx, "utf_big_string", TILEDB_STRING_UTF8); str_attr.set_cell_val_num(TILEDB_VAR_NUM); attrs.push_back(str_attr); - } - { - auto str_attr = Attribute(ctx, "utf_string2", TILEDB_STRING_UTF8); - str_attr.set_cell_val_num(TILEDB_VAR_NUM); - attrs.push_back(str_attr); - } - { - auto str_attr = Attribute(ctx, "tiledb_char", TILEDB_CHAR); - str_attr.set_cell_val_num(TILEDB_VAR_NUM); - attrs.push_back(str_attr); - } + } else if (offset == 32) { + attrs.insert( + attrs.end(), + {Attribute::create(ctx, "int8"), + Attribute::create(ctx, "int16"), + Attribute::create(ctx, "int32"), + Attribute::create(ctx, "int64"), + + Attribute::create(ctx, "uint8"), + Attribute::create(ctx, "uint16"), + Attribute::create(ctx, "uint32"), + Attribute::create(ctx, "uint64"), + + Attribute::create(ctx, "float32"), + Attribute::create(ctx, "float64")}); + + // must be constructed manually to get TILEDB_STRING_UTF8 type + { + auto str_attr = Attribute(ctx, "utf_string1", TILEDB_STRING_UTF8); + str_attr.set_cell_val_num(TILEDB_VAR_NUM); + attrs.push_back(str_attr); + } + { + auto str_attr = Attribute(ctx, "utf_string2", TILEDB_STRING_UTF8); + str_attr.set_cell_val_num(TILEDB_VAR_NUM); + attrs.push_back(str_attr); + } + { + auto str_attr = Attribute(ctx, "tiledb_char", TILEDB_CHAR); + str_attr.set_cell_val_num(TILEDB_VAR_NUM); + attrs.push_back(str_attr); + } - // must be constructed manually to get TILEDB_DATETIME_NS type - auto datetimens_attr = Attribute(ctx, "datetime_ns", TILEDB_DATETIME_NS); - attrs.push_back(datetimens_attr); + // must be constructed manually to get TILEDB_DATETIME_NS type + auto datetimens_attr = Attribute(ctx, "datetime_ns", TILEDB_DATETIME_NS); + attrs.push_back(datetimens_attr); + } FilterList filters(ctx); filters.add_filter({ctx, TILEDB_FILTER_LZ4}); @@ -217,9 +223,11 @@ void allocate_query_buffers(tiledb::Query* const query) { }; // namespace -void test_for_column_size(size_t col_size) { - std::string uri("test_arrow_io_" + std::to_string(col_size)); - CPPArrayFx _fx(uri, col_size); +void test_for_column_size(const size_t col_size, const uint8_t offset) { + std::string uri( + "test_arrow_io_" + std::to_string(col_size) + "_" + + std::to_string(offset)); + CPPArrayFx _fx(uri, col_size, offset); py::object py_data_source; py::object py_data_arrays; @@ -234,7 +242,8 @@ void test_for_column_size(size_t col_size) { unit_arrow = py::module::import("unit_arrow"); // this class generates random test data for each attribute - auto h_data_source = unit_arrow.attr("DataFactory"); + auto class_name = "DataFactory" + std::to_string(offset); + auto h_data_source = unit_arrow.attr(class_name.c_str()); py_data_source = h_data_source(py::int_(col_size)); py_data_names = py_data_source.attr("names"); py_data_arrays = py_data_source.attr("arrays"); @@ -248,7 +257,7 @@ void test_for_column_size(size_t col_size) { * Test write */ Config config; - config["sm.var_offsets.bitsize"] = 32; + config["sm.var_offsets.bitsize"] = offset; config["sm.var_offsets.mode"] = "elements"; config["sm.var_offsets.extra_element"] = "true"; Context ctx(config); @@ -303,14 +312,12 @@ void test_for_column_size(size_t col_size) { // However, there is an unexplained crash due to an early destructor // when both brace scopes are converted to SECTIONs. // SECTION("Test reading data back via ArrowAdapter into pyarrow arrays") - - // test both bitsize read modes - for (auto bitsize : {32, 64}) { + { /* * Test read */ Config config; - config["sm.var_offsets.bitsize"] = bitsize; + config["sm.var_offsets.bitsize"] = 64; config["sm.var_offsets.mode"] = "elements"; config["sm.var_offsets.extra_element"] = "true"; Context ctx(config); @@ -437,8 +444,9 @@ TEST_CASE("Arrow IO integration tests", "[arrow]") { #endif // do not use catch2 GENERATE here: it causes bad things to happen w/ python - uint64_t col_sizes[] = {0,1,2,3,4,11,103}; + uint64_t col_sizes[] = {0, 1, 2, 3, 4, 11, 103}; for (auto sz : col_sizes) { - test_for_column_size(sz); + test_for_column_size(sz, 32); + test_for_column_size(sz, 64); } } diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index fc643e84bc22..a2888cfe5801 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -44,6 +44,8 @@ #else #include "tiledb/sm/filesystem/posix.h" #endif +#include "tiledb/api/c_api/array_schema/array_schema_api_external.h" +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" #include "tiledb/api/c_api/buffer/buffer_api_internal.h" #include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/common/stdx_string.h" diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index bdd88904ddaf..341c6abfbf2b 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -2503,7 +2503,8 @@ TEST_CASE_METHOD( tiledb_array_schema_get_current_domain(ctx_, schema, &crd) == TILEDB_OK); uint32_t is_empty = 0; - REQUIRE(tiledb_current_domain_get_is_empty(crd, &is_empty) == TILEDB_OK); + REQUIRE( + tiledb_current_domain_get_is_empty(ctx_, crd, &is_empty) == TILEDB_OK); CHECK(is_empty == 1); REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); @@ -2537,7 +2538,7 @@ TEST_CASE_METHOD( tiledb_ndrectangle_t* ndr = nullptr; REQUIRE(tiledb_ndrectangle_alloc(ctx_, domain, &ndr) == TILEDB_OK); - REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(ctx_, crd, ndr) == TILEDB_OK); REQUIRE( tiledb_array_schema_set_current_domain(ctx_, schema, crd) == TILEDB_OK); @@ -2604,7 +2605,7 @@ TEST_CASE_METHOD( REQUIRE( tiledb_array_schema_get_current_domain(ctx_, schema, &crd) == TILEDB_OK); - REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_get_ndrectangle(ctx_, crd, &ndr) == TILEDB_OK); tiledb_range_t outrange; REQUIRE( tiledb_ndrectangle_get_range_from_name(ctx_, ndr, "d1", &outrange) == @@ -2672,7 +2673,7 @@ TEST_CASE_METHOD( REQUIRE( tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == TILEDB_OK); - REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(ctx_, crd, ndr) == TILEDB_OK); REQUIRE( tiledb_array_schema_set_current_domain(ctx_, schema, crd) == TILEDB_OK); @@ -2712,7 +2713,7 @@ TEST_CASE_METHOD( REQUIRE( tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == TILEDB_OK); - REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(ctx_, crd, ndr) == TILEDB_OK); REQUIRE( tiledb_array_schema_evolution_expand_current_domain(ctx_, evo, crd) == @@ -2752,7 +2753,7 @@ TEST_CASE_METHOD( REQUIRE( tiledb_array_schema_get_current_domain(ctx_, schema, &crd) == TILEDB_OK); - REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_get_ndrectangle(ctx_, crd, &ndr) == TILEDB_OK); tiledb_range_t outrange; REQUIRE( tiledb_ndrectangle_get_range_from_name(ctx_, ndr, "d1", &outrange) == diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 94336bf1003f..8875898ce624 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -230,7 +230,7 @@ void check_save_to_file() { ss << "rest.curl.buffer_size 524288\n"; ss << "rest.curl.verbose false\n"; ss << "rest.http_compressor any\n"; - ss << "rest.load_enumerations_on_array_open true\n"; + ss << "rest.load_enumerations_on_array_open false\n"; ss << "rest.load_metadata_on_array_open true\n"; ss << "rest.load_non_empty_domain_on_array_open true\n"; ss << "rest.retry_count 25\n"; diff --git a/test/src/unit-capi-enum_values.cc b/test/src/unit-capi-enum_values.cc index 24968c7e3409..5579c63ba4c4 100644 --- a/test/src/unit-capi-enum_values.cc +++ b/test/src/unit-capi-enum_values.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB Inc. + * @copyright Copyright (c) 2017-2024 TileDB Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -32,6 +32,7 @@ */ #include +#include "tiledb/api/c_api/array_schema/array_schema_api_external.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/enums/filter_type.h" diff --git a/test/src/unit-capi-fragment_info.cc b/test/src/unit-capi-fragment_info.cc index d3793239096c..2c12d6eefe77 100644 --- a/test/src/unit-capi-fragment_info.cc +++ b/test/src/unit-capi-fragment_info.cc @@ -1526,20 +1526,27 @@ TEST_CASE("C API: Test fragment info, dump", "[capi][fragment_info][dump]") { // Check dump const auto ver = std::to_string(tiledb::sm::constants::format_version); + const char* schema_name; + rc = tiledb_fragment_info_get_array_schema_name( + ctx, fragment_info, 0, &schema_name); + CHECK(rc == TILEDB_OK); std::string dump_str = std::string("- Fragment num: 3\n") + "- Unconsolidated metadata num: 3\n" + "- To vacuum num: 0\n" + "- Fragment #1:\n" + " > URI: " + written_frag_uri_1 + "\n" + - " > Type: dense\n" + " > Non-empty domain: [1, 6]\n" + - " > Size: 3202\n" + " > Cell num: 10\n" + - " > Timestamp range: [1, 1]\n" + " > Format version: " + ver + "\n" + + " > Schema name: " + schema_name + "\n" + " > Type: dense\n" + + " > Non-empty domain: [1, 6]\n" + " > Size: 3202\n" + + " > Cell num: 10\n" + " > Timestamp range: [1, 1]\n" + + " > Format version: " + ver + "\n" + " > Has consolidated metadata: no\n" + "- Fragment #2:\n" + - " > URI: " + written_frag_uri_2 + "\n" + " > Type: dense\n" + + " > URI: " + written_frag_uri_2 + "\n" + + " > Schema name: " + schema_name + "\n" + " > Type: dense\n" + " > Non-empty domain: [1, 4]\n" + " > Size: 3151\n" + " > Cell num: 5\n" + " > Timestamp range: [2, 2]\n" + " > Format version: " + ver + "\n" + " > Has consolidated metadata: no\n" + "- Fragment #3:\n" + - " > URI: " + written_frag_uri_3 + "\n" + " > Type: dense\n" + + " > URI: " + written_frag_uri_3 + "\n" + + " > Schema name: " + schema_name + "\n" + " > Type: dense\n" + " > Non-empty domain: [5, 6]\n" + " > Size: 3202\n" + " > Cell num: 10\n" + " > Timestamp range: [3, 3]\n" + " > Format version: " + ver + "\n" + @@ -1710,13 +1717,18 @@ TEST_CASE( CHECK(rc == TILEDB_OK); // Check dump + const char* schema_name; + rc = tiledb_fragment_info_get_array_schema_name( + ctx, fragment_info, 0, &schema_name); + CHECK(rc == TILEDB_OK); const auto ver = std::to_string(tiledb::sm::constants::format_version); std::string dump_str = std::string("- Fragment num: 1\n") + "- Unconsolidated metadata num: 1\n" + "- To vacuum num: 3\n" + "- To vacuum URIs:\n" + " > " + written_frag_uri_1 + "\n > " + written_frag_uri_2 + "\n > " + written_frag_uri_3 + "\n" + - "- Fragment #1:\n" + " > URI: " + uri + "\n" + " > Type: dense\n" + + "- Fragment #1:\n" + " > URI: " + uri + "\n" + + " > Schema name: " + schema_name + "\n" + " > Type: dense\n" + " > Non-empty domain: [1, 6]\n" + " > Size: 3208\n" + " > Cell num: 10\n" + " > Timestamp range: [1, 3]\n" + " > Format version: " + ver + "\n" + @@ -1821,13 +1833,18 @@ TEST_CASE( // Check dump const auto ver = std::to_string(tiledb::sm::constants::format_version); + const char* schema_name; + rc = tiledb_fragment_info_get_array_schema_name( + ctx, fragment_info, 0, &schema_name); + CHECK(rc == TILEDB_OK); std::string dump_str = std::string("- Fragment num: 1\n") + "- Unconsolidated metadata num: 1\n" + "- To vacuum num: 0\n" + "- Fragment #1:\n" + " > URI: " + written_frag_uri + "\n" + - " > Type: sparse\n" + " > Non-empty domain: [a, ddd]\n" + - " > Size: 3439\n" + " > Cell num: 4\n" + - " > Timestamp range: [1, 1]\n" + " > Format version: " + ver + "\n" + + " > Schema name: " + schema_name + "\n" + " > Type: sparse\n" + + " > Non-empty domain: [a, ddd]\n" + " > Size: 3439\n" + + " > Cell num: 4\n" + " > Timestamp range: [1, 1]\n" + + " > Format version: " + ver + "\n" + " > Has consolidated metadata: no\n"; FILE* gold_fout = fopen("gold_fout.txt", "w"); const char* dump = dump_str.c_str(); diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index 7ce02b75a089..fdf3ecf09fe5 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -60,13 +60,11 @@ struct ConsolidationWithTimestampsFx { // Functions. void set_legacy(); void create_sparse_array(bool allows_dups = false); - void create_sparse_array_v11(); void write_sparse( std::vector a1, std::vector dim1, std::vector dim2, uint64_t timestamp); - void write_sparse_v11(uint64_t timestamp); void consolidate_sparse(bool vacuum = false); void consolidate_sparse( uint64_t start_time, uint64_t end_time, bool vacuum = false); @@ -149,18 +147,6 @@ void ConsolidationWithTimestampsFx::create_sparse_array(bool allows_dups) { Array::create(SPARSE_ARRAY_NAME, schema); } -void ConsolidationWithTimestampsFx::create_sparse_array_v11() { - // Get the v11 sparse array. - std::string v11_arrays_dir = - std::string(TILEDB_TEST_INPUTS_DIR) + "/arrays/sparse_array_v11"; - REQUIRE( - tiledb_vfs_copy_dir( - ctx_.ptr().get(), - vfs_.ptr().get(), - v11_arrays_dir.c_str(), - SPARSE_ARRAY_NAME) == TILEDB_OK); -} - void ConsolidationWithTimestampsFx::write_sparse( std::vector a1, std::vector dim1, @@ -187,40 +173,6 @@ void ConsolidationWithTimestampsFx::write_sparse( array.close(); } -void ConsolidationWithTimestampsFx::write_sparse_v11(uint64_t timestamp) { - // Prepare cell buffers. - std::vector buffer_a1{0, 1, 2, 3}; - std::vector buffer_a2{0, 1, 3, 6}; - std::string buffer_var_a2("abbcccdddd"); - std::vector buffer_a3{0.1f, 0.2f, 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f}; - std::vector buffer_coords_dim1{1, 1, 1, 2}; - std::vector buffer_coords_dim2{1, 2, 4, 3}; - - // Open array. - Array array( - ctx_, - SPARSE_ARRAY_NAME, - TILEDB_WRITE, - tiledb::TemporalPolicy(tiledb::TimeTravel, timestamp)); - - // Create query. - Query query(ctx_, array, TILEDB_WRITE); - query.set_layout(TILEDB_GLOBAL_ORDER); - query.set_data_buffer("a1", buffer_a1); - query.set_data_buffer( - "a2", (void*)buffer_var_a2.c_str(), buffer_var_a2.size()); - query.set_offsets_buffer("a2", buffer_a2); - query.set_data_buffer("a3", buffer_a3); - query.set_data_buffer("d1", buffer_coords_dim1); - query.set_data_buffer("d2", buffer_coords_dim2); - - // Submit query - query.submit_and_finalize(); - - // Close array. - array.close(); -} - void ConsolidationWithTimestampsFx::consolidate_sparse(bool vacuum) { auto config = ctx_.config(); Array::consolidate(ctx_, SPARSE_ARRAY_NAME, &config); @@ -517,12 +469,12 @@ TEST_CASE_METHOD( } remove_sparse_array(); - create_sparse_array_v11(); + test::create_sparse_array_v11(ctx_.ptr().get(), SPARSE_ARRAY_NAME); // Write first fragment. - write_sparse_v11(1); + test::write_sparse_v11(ctx_.ptr().get(), SPARSE_ARRAY_NAME, 1); // Write second fragment. - write_sparse_v11(3); + test::write_sparse_v11(ctx_.ptr().get(), SPARSE_ARRAY_NAME, 3); // Consolidate. consolidate_sparse(); diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index fbb7fcd374a5..d2b5a81b26fe 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -80,14 +80,12 @@ struct DeletesFx { void create_dir(const std::string& path); void create_simple_array(const std::string& path); void create_sparse_array(bool allows_dups = false, bool encrypt = false); - void create_sparse_array_v11(); void write_sparse( std::vector a1, std::vector dim1, std::vector dim2, uint64_t timestamp, bool encrypt = false); - void write_sparse_v11(uint64_t timestamp); std::vector read_group(const tiledb::Group& group) const; void read_sparse( std::vector& a1, @@ -250,52 +248,6 @@ void DeletesFx::write_sparse( array->close(); } -void DeletesFx::create_sparse_array_v11() { - // Get the v11 sparse array. - std::string v11_arrays_dir = - std::string(TILEDB_TEST_INPUTS_DIR) + "/arrays/sparse_array_v11"; - REQUIRE( - tiledb_vfs_copy_dir( - ctx_.ptr().get(), - vfs_.ptr().get(), - v11_arrays_dir.c_str(), - array_name_.c_str()) == TILEDB_OK); -} - -void DeletesFx::write_sparse_v11(uint64_t timestamp) { - // Prepare cell buffers. - std::vector buffer_a1{0, 1, 2, 3}; - std::vector buffer_a2{0, 1, 3, 6}; - std::string buffer_var_a2("abbcccdddd"); - std::vector buffer_a3{0.1f, 0.2f, 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f}; - std::vector buffer_coords_dim1{1, 1, 1, 2}; - std::vector buffer_coords_dim2{1, 2, 4, 3}; - - // Open array. - Array array( - ctx_, - array_name_.c_str(), - TILEDB_WRITE, - tiledb::TemporalPolicy(tiledb::TimeTravel, timestamp)); - - // Create query. - Query query(ctx_, array, TILEDB_WRITE); - query.set_layout(TILEDB_GLOBAL_ORDER); - query.set_data_buffer("a1", buffer_a1); - query.set_data_buffer( - "a2", (void*)buffer_var_a2.c_str(), buffer_var_a2.size()); - query.set_offsets_buffer("a2", buffer_a2); - query.set_data_buffer("a3", buffer_a3); - query.set_data_buffer("d1", buffer_coords_dim1); - query.set_data_buffer("d2", buffer_coords_dim2); - - // Submit/finalize the query. - query.submit_and_finalize(); - - // Close array. - array.close(); -} - std::vector DeletesFx::read_group( const tiledb::Group& group) const { std::vector ret; @@ -1661,12 +1613,12 @@ TEST_CASE_METHOD( } remove_sparse_array(); - create_sparse_array_v11(); + test::create_sparse_array_v11(ctx_.ptr().get(), array_name_); // Write first fragment. - write_sparse_v11(1); + test::write_sparse_v11(ctx_.ptr().get(), array_name_, 1); // Write second fragment. - write_sparse_v11(3); + test::write_sparse_v11(ctx_.ptr().get(), array_name_, 3); // Consolidate. consolidate_sparse(); @@ -1947,8 +1899,8 @@ TEST_CASE_METHOD( auto array_name = array_name_ + "/"; // Write to v11 array - create_sparse_array_v11(); - write_sparse_v11(1); + test::create_sparse_array_v11(ctx_.ptr().get(), array_name_); + test::write_sparse_v11(ctx_.ptr().get(), array_name_, 1); std::string extraneous_file_path = array_name + "extraneous_file"; vfs_.touch(extraneous_file_path); diff --git a/test/src/unit-cppapi-fragment_info.cc b/test/src/unit-cppapi-fragment_info.cc index 9173cdfee84a..0cde19e60e48 100644 --- a/test/src/unit-cppapi-fragment_info.cc +++ b/test/src/unit-cppapi-fragment_info.cc @@ -1096,20 +1096,24 @@ TEST_CASE( // Check dump const auto ver = std::to_string(tiledb::sm::constants::format_version); + auto schema_name = fragment_info.array_schema_name(0); std::string dump_str = std::string("- Fragment num: 3\n") + "- Unconsolidated metadata num: 3\n" + "- To vacuum num: 0\n" + "- Fragment #1:\n" + " > URI: " + written_frag_uri_1 + "\n" + - " > Type: dense\n" + " > Non-empty domain: [1, 6]\n" + - " > Size: 3202\n" + " > Cell num: 10\n" + - " > Timestamp range: [1, 1]\n" + " > Format version: " + ver + "\n" + + " > Schema name: " + schema_name + "\n" + " > Type: dense\n" + + " > Non-empty domain: [1, 6]\n" + " > Size: 3202\n" + + " > Cell num: 10\n" + " > Timestamp range: [1, 1]\n" + + " > Format version: " + ver + "\n" + " > Has consolidated metadata: no\n" + "- Fragment #2:\n" + - " > URI: " + written_frag_uri_2 + "\n" + " > Type: dense\n" + + " > URI: " + written_frag_uri_2 + "\n" + + " > Schema name: " + schema_name + "\n" + " > Type: dense\n" + " > Non-empty domain: [1, 4]\n" + " > Size: 3151\n" + " > Cell num: 5\n" + " > Timestamp range: [2, 2]\n" + " > Format version: " + ver + "\n" + " > Has consolidated metadata: no\n" + "- Fragment #3:\n" + - " > URI: " + written_frag_uri_3 + "\n" + " > Type: dense\n" + + " > URI: " + written_frag_uri_3 + "\n" + + " > Schema name: " + schema_name + "\n" + " > Type: dense\n" + " > Non-empty domain: [5, 6]\n" + " > Size: 3202\n" + " > Cell num: 10\n" + " > Timestamp range: [3, 3]\n" + " > Format version: " + ver + "\n" + diff --git a/test/src/unit-cppapi-query-condition-enumerations.cc b/test/src/unit-cppapi-query-condition-enumerations.cc index 872fedab7c76..d1dd743824e2 100644 --- a/test/src/unit-cppapi-query-condition-enumerations.cc +++ b/test/src/unit-cppapi-query-condition-enumerations.cc @@ -31,6 +31,7 @@ */ #include +#include #include #include diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index 44403af07250..5a15fdc8d381 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB Inc. + * @copyright Copyright (c) 2023-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/test/src/unit-curl.cc b/test/src/unit-curl.cc index b8601e506e01..ee178c52082d 100644 --- a/test/src/unit-curl.cc +++ b/test/src/unit-curl.cc @@ -43,7 +43,8 @@ using namespace tiledb::sm; -TEST_CASE("CURL: Test curl's header parsing callback", "[curl]") { +TEST_CASE( + "CURL: Test curl's header parsing callback", "[curl][redirect-caching]") { // Initialize data that in real life scenario would be initialized by // RestClient char res_data[] = @@ -58,6 +59,7 @@ TEST_CASE("CURL: Test curl's header parsing callback", "[curl]") { std::unordered_map redirect_meta_; userdata.redirect_uri_map = &redirect_meta_; userdata.redirect_uri_map_lock = &redirect_mtx_; + userdata.should_cache_redirect = true; size_t result = write_header_callback(&res_data, size, count, &userdata); @@ -94,6 +96,35 @@ TEST_CASE("CURL: Test curl's header parsing callback", "[curl]") { "tiledb://my_username"); } +TEST_CASE( + "CURL: Test curl is not caching the region when requested so", + "[curl][redirect-caching]") { + // Initialize data that in real life scenario would be initialized by + // RestClient + char res_data[] = + "Location: https://test.url.domain/v1/arrays/testns/test_arr"; + size_t size = 1; + size_t count = sizeof(res_data); + + HeaderCbData userdata; + std::string ns_array = "testns:test_arr"; + userdata.uri = &ns_array; + std::mutex redirect_mtx_; + std::unordered_map redirect_meta_; + userdata.redirect_uri_map = &redirect_meta_; + userdata.redirect_uri_map_lock = &redirect_mtx_; + userdata.should_cache_redirect = false; + + size_t result = write_header_callback(&res_data, size, count, &userdata); + + // The return value should be equal to size * count + CHECK(result == size * count); + // The uri is not mutated inside the callback function + CHECK(*userdata.uri == ns_array); + // The redirect map holds a record of ns_array key and redirection url value + CHECK(userdata.redirect_uri_map->count(ns_array) == 0); +} + TEST_CASE( "RestClient: Remove trailing slash from rest_server_", "[rest-client]") { std::string rest_server = diff --git a/test/src/unit-current-domain-rest.cc b/test/src/unit-current-domain-rest.cc index 33112ac937de..210dd3641d95 100644 --- a/test/src/unit-current-domain-rest.cc +++ b/test/src/unit-current-domain-rest.cc @@ -128,7 +128,7 @@ void RestCurrentDomainFx::create_sparse_array(const std::string& array_name) { tiledb_ndrectangle_set_range_for_name(ctx_c_, ndr, "d2", &range_var) == TILEDB_OK); - REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(ctx_c_, crd, ndr) == TILEDB_OK); REQUIRE( tiledb_array_schema_set_current_domain(ctx_c_, array_schema, crd) == TILEDB_OK); @@ -169,7 +169,8 @@ TEST_CASE_METHOD( tiledb_array_schema_get_current_domain(ctx_c_, schema, &crd) == TILEDB_OK); - REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &ndr) == TILEDB_OK); + REQUIRE( + tiledb_current_domain_get_ndrectangle(ctx_c_, crd, &ndr) == TILEDB_OK); tiledb_range_t outrange; tiledb_range_t outrange_var; REQUIRE( @@ -239,7 +240,7 @@ TEST_CASE_METHOD( tiledb_ndrectangle_set_range_for_name(ctx_c_, ndr, "d2", &range_var) == TILEDB_OK); - REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(ctx_c_, crd, ndr) == TILEDB_OK); REQUIRE( tiledb_array_schema_evolution_expand_current_domain(ctx_c_, evo, crd) == TILEDB_OK); @@ -262,7 +263,8 @@ TEST_CASE_METHOD( tiledb_array_schema_get_current_domain(ctx_c_, schema, &crd) == TILEDB_OK); - REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &ndr) == TILEDB_OK); + REQUIRE( + tiledb_current_domain_get_ndrectangle(ctx_c_, crd, &ndr) == TILEDB_OK); tiledb_range_t outrange; REQUIRE( tiledb_ndrectangle_get_range_from_name(ctx_c_, ndr, "d1", &outrange) == diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 167a4b1096ca..482496bd76a3 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -30,6 +30,7 @@ * Tests the C++ API for enumeration related functions. */ +#include #include #include "test/support/src/mem_helpers.h" @@ -1119,6 +1120,69 @@ TEST_CASE_METHOD( REQUIRE(schema->is_enumeration_loaded("test_enmr") == true); } +TEST_CASE_METHOD( + EnumerationFx, + "Array - Load All Enumerations - All Schemas", + "[enumeration][array][load-all-enumerations][all-schemas]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + REQUIRE(schema->is_enumeration_loaded("test_enmr") == false); + std::string schema_name_1 = schema->name(); + + // Evolve once to add an enumeration. + auto ase = make_shared(HERE(), memory_tracker_); + std::vector var_values{"one", "two", "three"}; + auto var_enmr = create_enumeration( + var_values, false, Datatype::STRING_ASCII, "ase_var_enmr"); + ase->add_enumeration(var_enmr); + auto attr4 = make_shared(HERE(), "attr4", Datatype::UINT16); + attr4->set_enumeration_name("ase_var_enmr"); + CHECK_NOTHROW(ase->evolve_schema(schema)); + // Apply evolution to the array and reopen. + CHECK_NOTHROW(Array::evolve_array_schema( + ctx_.resources(), uri_, ase.get(), array->get_encryption_key())); + CHECK(array->reopen().ok()); + CHECK_NOTHROW(array->load_all_enumerations()); + auto all_schemas = array->array_schemas_all(); + schema = array->array_schema_latest_ptr(); + std::string schema_name_2 = schema->name(); + + // Check all schemas. + CHECK(all_schemas[schema_name_1]->is_enumeration_loaded("test_enmr") == true); + CHECK(all_schemas[schema_name_2]->is_enumeration_loaded("test_enmr") == true); + CHECK( + all_schemas[schema_name_2]->is_enumeration_loaded("ase_var_enmr") == + true); + + // Evolve a second time to drop an enumeration. + ase = make_shared(HERE(), memory_tracker_); + ase->drop_enumeration("test_enmr"); + ase->drop_attribute("attr1"); + CHECK_NOTHROW(ase->evolve_schema(schema)); + // Apply evolution to the array and reopen. + CHECK_NOTHROW(Array::evolve_array_schema( + ctx_.resources(), uri_, ase.get(), array->get_encryption_key())); + CHECK(array->reopen().ok()); + CHECK_NOTHROW(array->load_all_enumerations()); + all_schemas = array->array_schemas_all(); + schema = array->array_schema_latest_ptr(); + std::string schema_name_3 = schema->name(); + + // Check all schemas. + CHECK(all_schemas[schema_name_1]->is_enumeration_loaded("test_enmr") == true); + CHECK(all_schemas[schema_name_2]->is_enumeration_loaded("test_enmr") == true); + CHECK( + all_schemas[schema_name_2]->is_enumeration_loaded("ase_var_enmr") == + true); + CHECK_THROWS_WITH( + all_schemas[schema_name_3]->is_enumeration_loaded("test_enmr"), + Catch::Matchers::ContainsSubstring("No enumeration named")); + CHECK( + all_schemas[schema_name_3]->is_enumeration_loaded("ase_var_enmr") == + true); +} + TEST_CASE_METHOD( EnumerationFx, "Array - Load All Enumerations - Repeated", diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index b7368840d7a3..5480e86b0f43 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -32,6 +32,7 @@ #ifdef TILEDB_SERIALIZATION +#include "test/support/src/helpers.h" #include "test/support/src/mem_helpers.h" #include "test/support/tdb_catch.h" #include "tiledb/api/c_api/buffer/buffer_api_internal.h" @@ -41,6 +42,7 @@ #include "tiledb/sm/c_api/tiledb_serialization.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" @@ -67,6 +69,7 @@ struct RequestHandlerFx { Config cfg_; Context ctx_; EncryptionKey enc_key_; + shared_ptr schema_; }; struct HandleLoadArraySchemaRequestFx : RequestHandlerFx { @@ -75,11 +78,17 @@ struct HandleLoadArraySchemaRequestFx : RequestHandlerFx { } virtual shared_ptr create_schema() override; - shared_ptr call_handler( + + std::tuple< + shared_ptr, + std::unordered_map>> + call_handler( serialization::LoadArraySchemaRequest req, SerializationType stype); shared_ptr create_string_enumeration( std::string name, std::vector& values); + + shared_ptr schema_add_attribute(const std::string& attr_name); }; struct HandleQueryPlanRequestFx : RequestHandlerFx { @@ -116,15 +125,23 @@ struct HandleConsolidationPlanRequestFx : RequestHandlerFx { TEST_CASE_METHOD( HandleLoadArraySchemaRequestFx, - "tiledb_handle_load_array_schema_request - default request", + "tiledb_handle_load_array_schema_request - no enumerations", "[request_handler][load_array_schema][default]") { auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP); create_array(); - auto schema = - call_handler(serialization::LoadArraySchemaRequest(false), stype); + auto schema_response = + call_handler(serialization::LoadArraySchemaRequest(cfg_), stype); + auto schema = std::get<0>(schema_response); REQUIRE(schema->has_enumeration("enmr")); REQUIRE(schema->get_loaded_enumeration_names().size() == 0); + tiledb::test::schema_equiv(*schema, *schema_); + + // We did not evolve the schema so there should only be one. + auto all_schemas = std::get<1>(schema_response); + REQUIRE(all_schemas.size() == 1); + tiledb::test::schema_equiv( + *all_schemas.find(schema->name())->second, *schema_); } TEST_CASE_METHOD( @@ -134,12 +151,57 @@ TEST_CASE_METHOD( auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP); create_array(); - auto schema = - call_handler(serialization::LoadArraySchemaRequest(true), stype); + REQUIRE(cfg_.set("rest.load_enumerations_on_array_open", "true").ok()); + auto schema_response = + call_handler(serialization::LoadArraySchemaRequest(cfg_), stype); + auto schema = std::get<0>(schema_response); REQUIRE(schema->has_enumeration("enmr")); REQUIRE(schema->get_loaded_enumeration_names().size() == 1); REQUIRE(schema->get_loaded_enumeration_names()[0] == "enmr"); REQUIRE(schema->get_enumeration("enmr") != nullptr); + tiledb::test::schema_equiv(*schema, *schema_); + + // We did not evolve the schema so there should only be one. + auto all_schemas = std::get<1>(schema_response); + REQUIRE(all_schemas.size() == 1); + tiledb::test::schema_equiv( + *all_schemas.find(schema->name())->second, *schema_); +} + +TEST_CASE_METHOD( + HandleLoadArraySchemaRequestFx, + "tiledb_handle_load_array_schema_request - multiple schemas", + "[request_handler][load_array_schema][schema-evolution]") { + auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP); + std::string load_enums = GENERATE("true", "false"); + + create_array(); + + std::vector> all_schemas{schema_}; + all_schemas.push_back(schema_add_attribute("b")); + all_schemas.push_back(schema_add_attribute("c")); + all_schemas.push_back(schema_add_attribute("d")); + + REQUIRE(cfg_.set("rest.load_enumerations_on_array_open", load_enums).ok()); + auto schema_response = + call_handler(serialization::LoadArraySchemaRequest(cfg_), stype); + auto schema = std::get<0>(schema_response); + if (load_enums == "true") { + REQUIRE(schema->has_enumeration("enmr")); + REQUIRE(schema->get_loaded_enumeration_names().size() == 1); + REQUIRE(schema->get_loaded_enumeration_names()[0] == "enmr"); + REQUIRE(schema->get_enumeration("enmr") != nullptr); + } + // The latest schema should be equal to the last applied evolution. + tiledb::test::schema_equiv(*schema, *all_schemas.back()); + + // Validate schemas returned from the request in the order they were created. + auto r_all_schemas = std::get<1>(schema_response); + std::map> resp( + r_all_schemas.begin(), r_all_schemas.end()); + for (int i = 0; const auto& s : resp) { + tiledb::test::schema_equiv(*s.second, *all_schemas[i++]); + } } TEST_CASE_METHOD( @@ -346,7 +408,9 @@ TEST_CASE_METHOD( RequestHandlerFx::RequestHandlerFx(const std::string uri) : memory_tracker_(tiledb::test::create_test_memory_tracker()) , uri_(uri) - , ctx_(cfg_) { + , ctx_(cfg_) + , schema_(make_shared( + ArrayType::DENSE, ctx_.resources().ephemeral_memory_tracker())) { delete_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); } @@ -405,9 +469,28 @@ HandleLoadArraySchemaRequestFx::create_string_enumeration( tiledb::test::create_test_memory_tracker()); } +shared_ptr HandleLoadArraySchemaRequestFx::schema_add_attribute( + const std::string& attr_name) { + tiledb::Context ctx; + tiledb::ArraySchemaEvolution ase(ctx); + auto attr = tiledb::Attribute::create(ctx, attr_name); + ase.add_attribute(attr); + // Evolve and update the original schema member variable. + schema_ = ase.ptr()->array_schema_evolution_->evolve_schema(schema_); + // Apply the schema evolution. + Array::evolve_array_schema( + this->ctx_.resources(), + this->uri_, + ase.ptr()->array_schema_evolution_, + this->enc_key_); + + // Return the new evolved schema for validation. + return schema_; +} + shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { // Create a schema to serialize - auto schema = + schema_ = make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); auto dim = make_shared(HERE(), "dim1", Datatype::INT32, memory_tracker_); @@ -416,20 +499,23 @@ shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim)); - throw_if_not_ok(schema->set_domain(dom)); + throw_if_not_ok(schema_->set_domain(dom)); std::vector values = {"pig", "cow", "chicken", "dog", "cat"}; auto enmr = create_string_enumeration("enmr", values); - schema->add_enumeration(enmr); + schema_->add_enumeration(enmr); auto attr = make_shared(HERE(), "attr", Datatype::INT32); attr->set_enumeration_name("enmr"); - throw_if_not_ok(schema->add_attribute(attr)); + throw_if_not_ok(schema_->add_attribute(attr)); - return schema; + return schema_; } -shared_ptr HandleLoadArraySchemaRequestFx::call_handler( +std::tuple< + shared_ptr, + std::unordered_map>> +HandleLoadArraySchemaRequestFx::call_handler( serialization::LoadArraySchemaRequest req, SerializationType stype) { // If this looks weird, its because we're using the public C++ API to create // these objets instead of the internal APIs elsewhere in this test suite. @@ -451,7 +537,7 @@ shared_ptr HandleLoadArraySchemaRequestFx::call_handler( REQUIRE(rval == TILEDB_OK); return serialization::deserialize_load_array_schema_response( - stype, resp_buf->buffer(), memory_tracker_); + uri_, stype, resp_buf->buffer(), memory_tracker_); } shared_ptr HandleQueryPlanRequestFx::create_schema() { diff --git a/test/src/unit-sparse-global-order-reader.cc b/test/src/unit-sparse-global-order-reader.cc index 83c1e937572b..377fa149870f 100644 --- a/test/src/unit-sparse-global-order-reader.cc +++ b/test/src/unit-sparse-global-order-reader.cc @@ -44,6 +44,7 @@ #endif #include +#include using namespace tiledb; using namespace tiledb::test; diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index f8092eb224d4..74dbd148e234 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -46,6 +46,7 @@ #endif #include +#include using namespace tiledb::sm; using namespace tiledb::test; diff --git a/test/src/unit_arrow.py b/test/src/unit_arrow.py index a1e55d284688..4c2e342d3201 100644 --- a/test/src/unit_arrow.py +++ b/test/src/unit_arrow.py @@ -11,16 +11,10 @@ # Data generators # # ************************************************************************** # -# python 2 vs 3 compatibility -if sys.hexversion >= 0x3000000: - getchr = chr -else: - getchr = unichr - def gen_chr(max, printable=False): while True: # TODO we exclude 0x0 here because the key API does not embedded NULL - s = getchr(random.randrange(1, max)) + s = chr(random.randrange(1, max)) if printable and not s.isprintable(): continue if len(s) > 0: @@ -59,10 +53,10 @@ def rand_ascii_bytes(size=5, printable=False): return b''.join([gen_chr(127, printable).encode('utf-8') for _ in range(0,size)]) # ************************************************************************** # -# Test class # +# Test classes # # ************************************************************************** # -class DataFactory(): +class DataFactory32(): def __init__(self, col_size): self.results = {} self.col_size = col_size @@ -81,35 +75,65 @@ def create(self): for dt in (np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64): key = np.dtype(dt).name dtinfo = np.iinfo(dt) - self.data[key] = np.random.randint(dtinfo.min, dtinfo.max, size=col_size, dtype=dt) + self.data[key] = pa.array(np.random.randint(dtinfo.min, dtinfo.max, size=col_size, dtype=dt)) for dt in (np.float32, np.float64): key = np.dtype(dt).name - self.data[key] = np.random.rand(col_size).astype(dt) + self.data[key] = pa.array(np.random.rand(col_size).astype(dt)) # var-len (strings) - self.data['tiledb_char'] = np.array([rand_ascii_bytes(np.random.randint(1,100)) - for _ in range(col_size)]).astype("S1") - self.data['utf_string1'] = np.array([rand_utf8(np.random.randint(1, 100)) + self.data['tiledb_char'] = pa.array(np.array([rand_ascii_bytes(np.random.randint(1,100)) + for _ in range(col_size)]).astype("S1")) + utf_strings = np.array([rand_utf8(np.random.randint(1, 100)) for _ in range(col_size)]).astype("U0") + self.data['utf_string1'] = pa.array(utf_strings) - # another version with some important cells set to empty - self.data['utf_string2'] = np.array([rand_utf8(np.random.randint(0, 100)) - for _ in range(col_size)]).astype("U0") + # another version with some cells set to empty + utf_strings[np.random.randint(0, col_size, size=col_size//2)] = '' + self.data['utf_string2'] = pa.array(utf_strings) + + self.data['datetime_ns'] = pa.array(rand_datetime64_array(col_size)) + + ########################################################################## + + self.arrays = list(self.data.values()) + self.names = list(self.data.keys()) + + def import_result(self, name, c_array, c_schema): + self.results[name] = pa.Array._import_from_c(c_array, c_schema) + + def check(self): + for key,val in self.data.items(): + assert (key in self.results), "Expected key '{}' not found in results!".format(key) + + res_val = self.results[key] + assert_array_equal(val, res_val) - utf_string2 = self.data['utf_string2'] - for i in range(len(utf_string2)): - self.data['utf_string2'][i] = '' - range_start = len(utf_string2) - 1 - range_end = len(utf_string2) % 3 - for i in range(range_start, range_end, -1): - self.data['utf_string2'][i] = '' + return True + + +class DataFactory64(): + def __init__(self, col_size): + self.results = {} + self.col_size = col_size + self.create() + + def __len__(self): + if not self.data: + raise ValueError("Uninitialized data") + return len(self.data) + + def create(self): + # generate test data for all columns + col_size = self.col_size + self.data = {} - self.data['datetime_ns'] = rand_datetime64_array(col_size) + self.data['utf_big_string'] = pa.array([rand_utf8(np.random.randint(1, 100)) + for _ in range(col_size)], type=pa.large_utf8()) ########################################################################## - self.arrays = [pa.array(val) for val in self.data.values()] + self.arrays = list(self.data.values()) self.names = list(self.data.keys()) def import_result(self, name, c_array, c_schema): @@ -122,4 +146,4 @@ def check(self): res_val = self.results[key] assert_array_equal(val, res_val) - return True \ No newline at end of file + return True diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index dd497972016e..c8259245bb2a 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -1621,6 +1621,26 @@ void read_sparse_v11( tiledb_query_free(&query); } +void schema_equiv( + const sm::ArraySchema& schema1, const sm::ArraySchema& schema2) { + CHECK(schema1.array_type() == schema2.array_type()); + CHECK(schema1.attributes().size() == schema2.attributes().size()); + for (unsigned int i = 0; i < schema2.attribute_num(); i++) { + auto a = schema1.attribute(i); + auto b = schema2.attribute(i); + CHECK(a->cell_val_num() == b->cell_val_num()); + CHECK(a->name() == b->name()); + CHECK(a->type() == b->type()); + CHECK(a->nullable() == b->nullable()); + CHECK(a->get_enumeration_name() == b->get_enumeration_name()); + } + CHECK(schema1.capacity() == schema2.capacity()); + CHECK(schema1.cell_order() == schema2.cell_order()); + CHECK(schema1.tile_order() == schema2.tile_order()); + CHECK(schema1.allows_dups() == schema2.allows_dups()); + CHECK(schema1.array_uri().to_string() == schema2.array_uri().to_string()); +} + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); template void check_subarray( diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index dff048bc1a96..201821c09c6d 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -957,6 +957,15 @@ void write_sparse_v11( */ void read_sparse_v11( tiledb_ctx_t* ctx, const std::string& array_name, uint64_t timestamp); + +/** + * Helper function to test two array schemas are equivalent. + * + * @param schema1 Expected array schema. + * @param schema2 Actual array schema. + */ +void schema_equiv( + const sm::ArraySchema& schema1, const sm::ArraySchema& schema2); } // namespace tiledb::test #endif diff --git a/test/support/src/serialization_wrappers.cc b/test/support/src/serialization_wrappers.cc index 1ae4d58ea95c..6590523138c7 100644 --- a/test/support/src/serialization_wrappers.cc +++ b/test/support/src/serialization_wrappers.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,6 +32,7 @@ */ #include "test/support/src/helpers.h" +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_serialization.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" @@ -206,7 +207,7 @@ void tiledb_subarray_serialize( tiledb_array_schema_t* array_schema = nullptr; tiledb_array_get_schema(ctx, array, &array_schema); REQUIRE(tiledb::sm::serialization::subarray_to_capnp( - *(array_schema->array_schema_), (*subarray)->subarray_, &builder) + *(array_schema->array_schema()), (*subarray)->subarray_, &builder) .ok()); // Deserialize tiledb_subarray_t* deserialized_subarray; diff --git a/tiledb/api/c_api/CMakeLists.txt b/tiledb/api/c_api/CMakeLists.txt index e511ee25a457..a5943f9a5d80 100644 --- a/tiledb/api/c_api/CMakeLists.txt +++ b/tiledb/api/c_api/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022-2023 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -52,6 +52,9 @@ target_include_directories(export INTERFACE ${TILEDB_SOURCE_ROOT}/tiledb/sm/c_ap # `array`: no dependencies add_subdirectory(array) +# `array_schema`: depends on `context` +add_subdirectory(array_schema) + # `buffer`: no dependencies add_subdirectory(buffer) diff --git a/tiledb/api/c_api/array/CMakeLists.txt b/tiledb/api/c_api/array/CMakeLists.txt index 01b40189feb5..042a1fdd8218 100644 --- a/tiledb/api/c_api/array/CMakeLists.txt +++ b/tiledb/api/c_api/array/CMakeLists.txt @@ -25,5 +25,18 @@ # include(common NO_POLICY_SCOPE) +include(object_library) + +list(APPEND SOURCES + array_api.cc +) +gather_sources(${SOURCES}) + +commence(object_library capi_array_stub) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_object_libraries(array) + this_target_object_libraries(capi_context_stub) +conclude(object_library) add_test_subdirectory() diff --git a/tiledb/api/c_api/array/array_api.cc b/tiledb/api/c_api/array/array_api.cc new file mode 100644 index 000000000000..91601b9e5039 --- /dev/null +++ b/tiledb/api/c_api/array/array_api.cc @@ -0,0 +1,101 @@ +/** + * @file tiledb/api/c_api/array/array_api.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines C API functions for the array section. + */ + +#include "array_api_internal.h" +#include "tiledb/sm/c_api/tiledb.h" + +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api_support/c_api_support.h" +#include "tiledb/sm/array/array_directory.h" +#include "tiledb/sm/array_schema/array_schema_operations.h" +#include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/rest/rest_client.h" + +namespace tiledb::api { + +capi_return_t tiledb_array_schema_load( + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_array_schema_t** array_schema) { + // Create array schema + ensure_context_is_valid(ctx); + ensure_output_pointer_is_valid(array_schema); + + // Use a default constructed config to load the schema with default options. + *array_schema = tiledb_array_schema_t::make_handle( + load_array_schema(ctx->context(), sm::URI(array_uri), sm::Config())); + + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_load_with_config( + tiledb_ctx_t* ctx, + tiledb_config_t* config, + const char* array_uri, + tiledb_array_schema_t** array_schema) { + ensure_context_is_valid(ctx); + ensure_config_is_valid(config); + ensure_output_pointer_is_valid(array_schema); + + // Use passed config or context config to load the schema with set options. + *array_schema = tiledb_array_schema_t::make_handle(load_array_schema( + ctx->context(), + sm::URI(array_uri), + config ? config->config() : ctx->config())); + + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_with_context; + +CAPI_INTERFACE( + array_schema_load, + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_array_schema_t** array_schema) { + return api_entry_with_context( + ctx, array_uri, array_schema); +} + +CAPI_INTERFACE( + array_schema_load_with_config, + tiledb_ctx_t* ctx, + tiledb_config_t* config, + const char* array_uri, + tiledb_array_schema_t** array_schema) { + return api_entry_with_context< + tiledb::api::tiledb_array_schema_load_with_config>( + ctx, config, array_uri, array_schema); +} diff --git a/tiledb/api/c_api/array/array_api_external.h b/tiledb/api/c_api/array/array_api_external.h new file mode 100644 index 000000000000..e273e07fbfe7 --- /dev/null +++ b/tiledb/api/c_api/array/array_api_external.h @@ -0,0 +1,104 @@ +/** + * @file tiledb/api/c_api/array/array_api_external.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the Array C API for TileDB. + */ + +#ifndef TILEDB_CAPI_ARRAY_EXTERNAL_H +#define TILEDB_CAPI_ARRAY_EXTERNAL_H + +#include "../api_external_common.h" +#include "../array_schema/array_schema_api_external.h" +#include "../context/context_api_external.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Retrieves the latest schema of an array from the disk, creating an array + * schema struct. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_t* array_schema; + * tiledb_array_schema_load(ctx, "s3://tiledb_bucket/my_array", &array_schema); + * // Make sure to free the array schema in the end + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_uri The array whose schema will be retrieved. + * @param[out] array_schema The array schema to be retrieved, or `NULL` upon + * error. + * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_load( + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; + +/** + * Retrieves the latest schema of an array, creating an array schema struct. + * Options to load additional features are read from the provided + * tiledb_config_t* instance. If the provided config is nullptr, the config from + * `ctx` is used instead. + * + * Currently supported options to be read from the config: + * - rest.load_enumerations_on_array_open - boolean + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_t* array_schema; + * tiledb_array_schema_load_with_config( + * ctx, + * config, + * "s3://tiledb_bucket/my_array", + * &array_schema); + * // Make sure to free the array schema in the end + * @endcode + * + * @param ctx The TileDB context. + * @param config The TileDB config. + * @param array_uri The array whose schema will be retrieved. + * @param array_schema The array schema to be retrieved, or `NULL` upon error. + * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_load_with_config( + tiledb_ctx_t* ctx, + tiledb_config_t* config, + const char* array_uri, + tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_ARRAY_EXTERNAL_H diff --git a/tiledb/api/c_api/array/array_api_internal.h b/tiledb/api/c_api/array/array_api_internal.h new file mode 100644 index 000000000000..e6b6e03935f2 --- /dev/null +++ b/tiledb/api/c_api/array/array_api_internal.h @@ -0,0 +1,40 @@ +/** + * @file tiledb/api/c_api/array/array_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the internals of the array section of the C API. + */ + +#ifndef TILEDB_CAPI_ARRAY_INTERNAL_H +#define TILEDB_CAPI_ARRAY_INTERNAL_H + +#include "array_api_external.h" + +#include "tiledb/common/common.h" + +#endif // TILEDB_CAPI_ARRAY_INTERNAL_H diff --git a/tiledb/api/c_api/array/test/CMakeLists.txt b/tiledb/api/c_api/array/test/CMakeLists.txt index 95d84a29b2bf..afa865016d93 100644 --- a/tiledb/api/c_api/array/test/CMakeLists.txt +++ b/tiledb/api/c_api/array/test/CMakeLists.txt @@ -32,7 +32,7 @@ find_package(Catch2 REQUIRED) # Maturity # # This unit links against the whole library out of necessity. There's not -# yet an `array` object library, much less a C API handle one. +# yet a _complete_ `array` C API handle object library. # commence(unit_test capi_array) this_target_sources(unit_capi_array.cc) diff --git a/tiledb/sm/crypto/test/compile_crypto_main.cc b/tiledb/api/c_api/array/test/compile_capi_array_stub_main.cc similarity index 84% rename from tiledb/sm/crypto/test/compile_crypto_main.cc rename to tiledb/api/c_api/array/test/compile_capi_array_stub_main.cc index ea3eef1c5794..8907a6f43e3d 100644 --- a/tiledb/sm/crypto/test/compile_crypto_main.cc +++ b/tiledb/api/c_api/array/test/compile_capi_array_stub_main.cc @@ -1,11 +1,12 @@ /** - * @file compile_crypto_main.cc + * @file + * tiledb/api/c_api/array/test/compile_capi_array_stub_main.cc * * @section LICENSE * * The MIT License * - * @copyright Copyright (c) 2021 TileDB, Inc. + * @copyright Copyright (c) 2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,9 +27,9 @@ * THE SOFTWARE. */ -#include "../crypto.h" +#include "../array_api_external.h" int main() { - (void)sizeof(tiledb::sm::Crypto); + tiledb_array_schema_load(nullptr, nullptr, nullptr); return 0; } diff --git a/tiledb/api/c_api/array/test/unit_capi_array.cc b/tiledb/api/c_api/array/test/unit_capi_array.cc index b521aacb4c69..29406b849014 100644 --- a/tiledb/api/c_api/array/test/unit_capi_array.cc +++ b/tiledb/api/c_api/array/test/unit_capi_array.cc @@ -33,7 +33,11 @@ #include "tiledb/api/c_api_test_support/testsupport_capi_context.h" #include "tiledb/sm/c_api/tiledb.h" -using tiledb::api::test_support::ordinary_context; +#include "../array_api_external.h" +#include "../array_api_internal.h" + +using namespace tiledb::api::test_support; + const char* TEST_URI = "unit_capi_array"; TEST_CASE( @@ -78,3 +82,26 @@ TEST_CASE( CHECK(tiledb_status(rc) == TILEDB_ERR); } } + +TEST_CASE( + "C API: tiledb_array_schema_load argument validation", "[capi][array]") { + capi_return_t rc; + ordinary_context ctx{}; + const char* array_uri = "array_uri"; + tiledb_array_schema_handle_t* schema{}; + /* + * No "success" section here; too much overhead to set up. + */ + SECTION("null context") { + rc = tiledb_array_schema_load(nullptr, array_uri, &schema); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null array_uri") { + rc = tiledb_array_schema_load(ctx.context, nullptr, &schema); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null schema") { + rc = tiledb_array_schema_load(ctx.context, array_uri, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} diff --git a/tiledb/api/c_api/array_schema/CMakeLists.txt b/tiledb/api/c_api/array_schema/CMakeLists.txt new file mode 100644 index 000000000000..4beb4b49d9f8 --- /dev/null +++ b/tiledb/api/c_api/array_schema/CMakeLists.txt @@ -0,0 +1,42 @@ +# +# tiledb/api/c_api/array_schema/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) +include(object_library) + +list(APPEND SOURCES + array_schema_api.cc +) +gather_sources(${SOURCES}) + +commence(object_library capi_array_schema_stub) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_object_libraries(array_schema) + this_target_object_libraries(capi_context_stub) +conclude(object_library) + +add_test_subdirectory() diff --git a/tiledb/api/c_api/array_schema/array_schema_api.cc b/tiledb/api/c_api/array_schema/array_schema_api.cc new file mode 100644 index 000000000000..3191dc75c12b --- /dev/null +++ b/tiledb/api/c_api/array_schema/array_schema_api.cc @@ -0,0 +1,740 @@ +/** + * @file tiledb/api/c_api/array_schema/array_schema_api.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines C API functions for the array schema section. + */ + +#include "array_schema_api_deprecated.h" +#include "array_schema_api_experimental.h" +#include "array_schema_api_internal.h" + +#include "tiledb/api/c_api/attribute/attribute_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_internal.h" +#include "tiledb/api/c_api/domain/domain_api_internal.h" +#include "tiledb/api/c_api/enumeration/enumeration_api_internal.h" +#include "tiledb/api/c_api/filter_list/filter_list_api_internal.h" +#include "tiledb/api/c_api/string/string_api_internal.h" +#include "tiledb/api/c_api_support/c_api_support.h" +#include "tiledb/common/memory_tracker.h" + +namespace tiledb::api { + +capi_return_t tiledb_array_type_to_str( + tiledb_array_type_t array_type, const char** str) { + const auto& strval = + tiledb::sm::array_type_str((tiledb::sm::ArrayType)array_type); + *str = strval.c_str(); + return strval.empty() ? TILEDB_ERR : TILEDB_OK; +} + +capi_return_t tiledb_array_type_from_str( + const char* str, tiledb_array_type_t* array_type) { + tiledb::sm::ArrayType val = tiledb::sm::ArrayType::DENSE; + if (!tiledb::sm::array_type_enum(str, &val).ok()) { + return TILEDB_ERR; + } + *array_type = (tiledb_array_type_t)val; + return TILEDB_OK; +} + +capi_return_t tiledb_layout_to_str(tiledb_layout_t layout, const char** str) { + const auto& strval = tiledb::sm::layout_str((tiledb::sm::Layout)layout); + *str = strval.c_str(); + return strval.empty() ? TILEDB_ERR : TILEDB_OK; +} + +capi_return_t tiledb_layout_from_str(const char* str, tiledb_layout_t* layout) { + tiledb::sm::Layout val = tiledb::sm::Layout::ROW_MAJOR; + if (!tiledb::sm::layout_enum(str, &val).ok()) { + return TILEDB_ERR; + } + *layout = (tiledb_layout_t)val; + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_alloc( + tiledb_ctx_t* ctx, + tiledb_array_type_t array_type, + tiledb_array_schema_t** array_schema) { + ensure_output_pointer_is_valid(array_schema); + + // Create ArraySchema object + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(MemoryTrackerType::ARRAY_CREATE); + *array_schema = tiledb_array_schema_t::make_handle( + static_cast(array_type), memory_tracker); + + return TILEDB_OK; +} + +void tiledb_array_schema_free(tiledb_array_schema_t** array_schema) { + ensure_output_pointer_is_valid(array_schema); + ensure_array_schema_is_valid(*array_schema); + tiledb_array_schema_t::break_handle(*array_schema); +} + +capi_return_t tiledb_array_schema_add_attribute( + tiledb_array_schema_t* array_schema, tiledb_attribute_t* attr) { + ensure_array_schema_is_valid(array_schema); + ensure_attribute_is_valid(attr); + + /** Note: The call to make_shared creates a copy of the attribute and + * the user-visible handle to the attr no longer refers to the same object + * that's in the array_schema. + **/ + throw_if_not_ok(array_schema->add_attribute(attr->copy_attribute())); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_set_allows_dups( + tiledb_array_schema_t* array_schema, int allows_dups) { + ensure_array_schema_is_valid(array_schema); + throw_if_not_ok(array_schema->set_allows_dups(allows_dups)); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_allows_dups( + tiledb_array_schema_t* array_schema, int* allows_dups) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(allows_dups); + *allows_dups = (int)array_schema->allows_dups(); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_version( + tiledb_array_schema_t* array_schema, uint32_t* version) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(version); + *version = (uint32_t)array_schema->version(); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_set_domain( + tiledb_array_schema_t* array_schema, tiledb_domain_t* domain) { + ensure_array_schema_is_valid(array_schema); + ensure_domain_is_valid(domain); + throw_if_not_ok(array_schema->set_domain(domain->copy_domain())); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_set_capacity( + tiledb_array_schema_t* array_schema, uint64_t capacity) { + ensure_array_schema_is_valid(array_schema); + array_schema->set_capacity(capacity); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_set_cell_order( + tiledb_array_schema_t* array_schema, tiledb_layout_t cell_order) { + ensure_array_schema_is_valid(array_schema); + throw_if_not_ok(array_schema->set_cell_order( + static_cast(cell_order))); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_set_tile_order( + tiledb_array_schema_t* array_schema, tiledb_layout_t tile_order) { + ensure_array_schema_is_valid(array_schema); + throw_if_not_ok(array_schema->set_tile_order( + static_cast(tile_order))); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_timestamp_range( + tiledb_array_schema_t* array_schema, uint64_t* lo, uint64_t* hi) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(lo); + ensure_output_pointer_is_valid(hi); + + auto timestamp_range = array_schema->timestamp_range(); + *lo = std::get<0>(timestamp_range); + *hi = std::get<1>(timestamp_range); + + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_add_enumeration( + tiledb_array_schema_t* array_schema, tiledb_enumeration_t* enumeration) { + ensure_array_schema_is_valid(array_schema); + ensure_enumeration_is_valid(enumeration); + array_schema->add_enumeration(enumeration->copy()); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_set_coords_filter_list( + tiledb_array_schema_t* array_schema, tiledb_filter_list_t* filter_list) { + ensure_array_schema_is_valid(array_schema); + ensure_filter_list_is_valid(filter_list); + throw_if_not_ok( + array_schema->set_coords_filter_pipeline(filter_list->pipeline())); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_set_offsets_filter_list( + tiledb_array_schema_t* array_schema, tiledb_filter_list_t* filter_list) { + ensure_array_schema_is_valid(array_schema); + ensure_filter_list_is_valid(filter_list); + throw_if_not_ok(array_schema->set_cell_var_offsets_filter_pipeline( + filter_list->pipeline())); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_set_validity_filter_list( + tiledb_array_schema_t* array_schema, tiledb_filter_list_t* filter_list) { + ensure_array_schema_is_valid(array_schema); + ensure_filter_list_is_valid(filter_list); + throw_if_not_ok( + array_schema->set_cell_validity_filter_pipeline(filter_list->pipeline())); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_check( + tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema) { + ensure_array_schema_is_valid(array_schema); + array_schema->check(ctx->resources().config()); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_array_type( + const tiledb_array_schema_t* array_schema, + tiledb_array_type_t* array_type) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(array_type); + *array_type = static_cast(array_schema->array_type()); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_capacity( + const tiledb_array_schema_t* array_schema, uint64_t* capacity) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(capacity); + *capacity = array_schema->capacity(); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_cell_order( + const tiledb_array_schema_t* array_schema, tiledb_layout_t* cell_order) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(cell_order); + *cell_order = static_cast(array_schema->cell_order()); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_coords_filter_list( + tiledb_array_schema_t* array_schema, tiledb_filter_list_t** filter_list) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(filter_list); + // Copy-construct a separate FilterPipeline object + *filter_list = tiledb_filter_list_t::make_handle( + sm::FilterPipeline{array_schema->coords_filters()}); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_offsets_filter_list( + tiledb_array_schema_t* array_schema, tiledb_filter_list_t** filter_list) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(filter_list); + // Copy-construct a separate FilterPipeline object + *filter_list = tiledb_filter_list_t::make_handle( + sm::FilterPipeline{array_schema->cell_var_offsets_filters()}); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_validity_filter_list( + tiledb_array_schema_t* array_schema, tiledb_filter_list_t** filter_list) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(filter_list); + // Copy-construct a separate FilterPipeline object + *filter_list = tiledb_filter_list_t::make_handle( + sm::FilterPipeline{array_schema->cell_validity_filters()}); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_domain( + const tiledb_array_schema_t* array_schema, tiledb_domain_t** domain) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(domain); + *domain = tiledb_domain_handle_t::make_handle(array_schema->shared_domain()); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_tile_order( + const tiledb_array_schema_t* array_schema, tiledb_layout_t* tile_order) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(tile_order); + *tile_order = static_cast(array_schema->tile_order()); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_attribute_num( + const tiledb_array_schema_t* array_schema, uint32_t* attribute_num) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(attribute_num); + *attribute_num = array_schema->attribute_num(); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_dump( + const tiledb_array_schema_t* array_schema, FILE* out) { + // Note: this API is deprecated. + ensure_array_schema_is_valid(array_schema); + ensure_cstream_handle_is_valid(out); + + std::stringstream ss; + ss << *array_schema->array_schema(); + size_t r = fwrite(ss.str().c_str(), sizeof(char), ss.str().size(), out); + if (r != ss.str().size()) { + throw CAPIException( + "Error writing array schema " + array_schema->array_uri().to_string() + + " to file"); + } + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_dump_str( + const tiledb_array_schema_t* array_schema, tiledb_string_t** out) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(out); + + std::stringstream ss; + ss << *array_schema->array_schema(); + *out = tiledb_string_handle_t::make_handle(ss.str()); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_attribute_from_index( + const tiledb_array_schema_t* array_schema, + uint32_t index, + tiledb_attribute_t** attr) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(attr); + + uint32_t attribute_num = array_schema->attribute_num(); + if (attribute_num == 0) { + *attr = nullptr; + return TILEDB_OK; + } + if (index >= attribute_num) { + std::ostringstream errmsg; + errmsg << "Attribute index: " << index << " out of bounds given " + << attribute_num << " attributes in array " + << array_schema->array_uri().to_string(); + throw CAPIException(errmsg.str()); + } + + auto found_attr = array_schema->shared_attribute(index); + if (!found_attr) { + throw CAPIException("Attribute not found, but index is valid!"); + } + *attr = tiledb_attribute_handle_t::make_handle(found_attr); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_attribute_from_name( + const tiledb_array_schema_t* array_schema, + const char* name, + tiledb_attribute_t** attr) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(attr); + + uint32_t attribute_num = array_schema->attribute_num(); + if (attribute_num == 0) { + *attr = nullptr; + return TILEDB_OK; + } + std::string name_string(name); + auto found_attr = array_schema->shared_attribute(name_string); + if (!found_attr) { + throw CAPIException( + std::string("Attribute name: ") + + (name_string.empty() ? "" : name) + + " does not exist for array " + array_schema->array_uri().to_string()); + } + *attr = tiledb_attribute_handle_t::make_handle(found_attr); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_has_attribute( + const tiledb_array_schema_t* array_schema, + const char* name, + int32_t* has_attr) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(has_attr); + + bool b; + throw_if_not_ok(array_schema->has_attribute(name, &b)); + *has_attr = b ? 1 : 0; + + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_set_current_domain( + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t* current_domain) { + ensure_array_schema_is_valid(array_schema); + ensure_handle_is_valid(current_domain); + array_schema->set_current_domain(current_domain->current_domain()); + return TILEDB_OK; +} + +capi_return_t tiledb_array_schema_get_current_domain( + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t** current_domain) { + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(current_domain); + + // There is always a current domain on an ArraySchema instance, + // when none was set explicitly, there is an empty current domain. + *current_domain = tiledb_current_domain_handle_t::make_handle( + array_schema->get_current_domain()); + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_context; +using tiledb::api::api_entry_plain; +using tiledb::api::api_entry_void; +using tiledb::api::api_entry_with_context; + +CAPI_INTERFACE( + array_type_to_str, tiledb_array_type_t array_type, const char** str) { + return api_entry_plain( + array_type, str); +} + +CAPI_INTERFACE( + array_type_from_str, const char* str, tiledb_array_type_t* array_type) { + return api_entry_plain( + str, array_type); +} + +CAPI_INTERFACE(layout_to_str, tiledb_layout_t layout, const char** str) { + return api_entry_plain(layout, str); +} + +CAPI_INTERFACE(layout_from_str, const char* str, tiledb_layout_t* layout) { + return api_entry_plain(str, layout); +} + +CAPI_INTERFACE( + array_schema_alloc, + tiledb_ctx_t* ctx, + tiledb_array_type_t array_type, + tiledb_array_schema_t** array_schema) { + return api_entry_with_context( + ctx, array_type, array_schema); +} + +CAPI_INTERFACE_VOID(array_schema_free, tiledb_array_schema_t** array_schema) { + return api_entry_void(array_schema); +} + +CAPI_INTERFACE( + array_schema_add_attribute, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_attribute_t* attr) { + return api_entry_context( + ctx, array_schema, attr); +} + +CAPI_INTERFACE( + array_schema_set_allows_dups, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + int allows_dups) { + return api_entry_context( + ctx, array_schema, allows_dups); +} + +CAPI_INTERFACE( + array_schema_get_allows_dups, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + int* allows_dups) { + return api_entry_context( + ctx, array_schema, allows_dups); +} + +CAPI_INTERFACE( + array_schema_get_version, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + uint32_t* version) { + return api_entry_context( + ctx, array_schema, version); +} + +CAPI_INTERFACE( + array_schema_set_domain, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_domain_t* domain) { + return api_entry_context( + ctx, array_schema, domain); +} + +CAPI_INTERFACE( + array_schema_set_capacity, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + uint64_t capacity) { + return api_entry_context( + ctx, array_schema, capacity); +} + +CAPI_INTERFACE( + array_schema_set_cell_order, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_layout_t cell_order) { + return api_entry_context( + ctx, array_schema, cell_order); +} + +CAPI_INTERFACE( + array_schema_set_tile_order, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_layout_t tile_order) { + return api_entry_context( + ctx, array_schema, tile_order); +} + +CAPI_INTERFACE( + array_schema_timestamp_range, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + uint64_t* lo, + uint64_t* hi) { + return api_entry_context( + ctx, array_schema, lo, hi); +} + +CAPI_INTERFACE( + array_schema_add_enumeration, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) { + return api_entry_context( + ctx, array_schema, enumeration); +} + +CAPI_INTERFACE( + array_schema_set_coords_filter_list, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t* filter_list) { + return api_entry_context< + tiledb::api::tiledb_array_schema_set_coords_filter_list>( + ctx, array_schema, filter_list); +} + +CAPI_INTERFACE( + array_schema_set_offsets_filter_list, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t* filter_list) { + return api_entry_context< + tiledb::api::tiledb_array_schema_set_offsets_filter_list>( + ctx, array_schema, filter_list); +} + +CAPI_INTERFACE( + array_schema_set_validity_filter_list, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t* filter_list) { + return api_entry_context< + tiledb::api::tiledb_array_schema_set_validity_filter_list>( + ctx, array_schema, filter_list); +} + +CAPI_INTERFACE( + array_schema_check, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema) { + return api_entry_with_context( + ctx, array_schema); +} + +CAPI_INTERFACE( + array_schema_get_array_type, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_array_type_t* array_type) { + return api_entry_context( + ctx, array_schema, array_type); +} + +CAPI_INTERFACE( + array_schema_get_capacity, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + uint64_t* capacity) { + return api_entry_context( + ctx, array_schema, capacity); +} + +CAPI_INTERFACE( + array_schema_get_cell_order, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_layout_t* cell_order) { + return api_entry_context( + ctx, array_schema, cell_order); +} + +CAPI_INTERFACE( + array_schema_get_coords_filter_list, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t** filter_list) { + return api_entry_context< + tiledb::api::tiledb_array_schema_get_coords_filter_list>( + ctx, array_schema, filter_list); +} + +CAPI_INTERFACE( + array_schema_get_offsets_filter_list, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t** filter_list) { + return api_entry_context< + tiledb::api::tiledb_array_schema_get_offsets_filter_list>( + ctx, array_schema, filter_list); +} + +CAPI_INTERFACE( + array_schema_get_validity_filter_list, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t** filter_list) { + return api_entry_context< + tiledb::api::tiledb_array_schema_get_validity_filter_list>( + ctx, array_schema, filter_list); +} + +CAPI_INTERFACE( + array_schema_get_domain, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_domain_t** domain) { + return api_entry_context( + ctx, array_schema, domain); +} + +CAPI_INTERFACE( + array_schema_get_tile_order, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_layout_t* tile_order) { + return api_entry_context( + ctx, array_schema, tile_order); +} + +CAPI_INTERFACE( + array_schema_get_attribute_num, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + uint32_t* attribute_num) { + return api_entry_context( + ctx, array_schema, attribute_num); +} + +CAPI_INTERFACE( + array_schema_dump, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + FILE* out) { + return api_entry_context( + ctx, array_schema, out); +} + +CAPI_INTERFACE( + array_schema_dump_str, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_string_t** out) { + return api_entry_context( + ctx, array_schema, out); +} + +CAPI_INTERFACE( + array_schema_get_attribute_from_index, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + uint32_t index, + tiledb_attribute_t** attr) { + return api_entry_context< + tiledb::api::tiledb_array_schema_get_attribute_from_index>( + ctx, array_schema, index, attr); +} + +CAPI_INTERFACE( + array_schema_get_attribute_from_name, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + const char* name, + tiledb_attribute_t** attr) { + return api_entry_context< + tiledb::api::tiledb_array_schema_get_attribute_from_name>( + ctx, array_schema, name, attr); +} + +CAPI_INTERFACE( + array_schema_has_attribute, + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + const char* name, + int32_t* has_attr) { + return api_entry_context( + ctx, array_schema, name, has_attr); +} + +CAPI_INTERFACE( + array_schema_set_current_domain, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t* current_domain) { + return api_entry_context( + ctx, array_schema, current_domain); +} + +CAPI_INTERFACE( + array_schema_get_current_domain, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t** current_domain) { + return api_entry_context( + ctx, array_schema, current_domain); +} diff --git a/tiledb/api/c_api/array_schema/array_schema_api_deprecated.h b/tiledb/api/c_api/array_schema/array_schema_api_deprecated.h new file mode 100644 index 000000000000..7821b06f331c --- /dev/null +++ b/tiledb/api/c_api/array_schema/array_schema_api_deprecated.h @@ -0,0 +1,68 @@ +/** + * @file tiledb/api/c_api/array_schema/array_schema_api_deprecated.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the deprecated array schema section of the TileDB C API. + */ + +#ifndef TILEDB_CAPI_ARRAY_SCHEMA_DEPRECATED_H +#define TILEDB_CAPI_ARRAY_SCHEMA_DEPRECATED_H + +#include "../api_external_common.h" +#include "array_schema_api_external.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Dumps the array schema in ASCII format in the selected file output. + * + * **Example:** + * + * The following prints the array schema dump in standard output. + * + * @code{.c} + * tiledb_array_schema_dump(ctx, array_schema, stdout); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] out The output handle. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT capi_return_t tiledb_array_schema_dump( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + FILE* out) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_ARRAY_SCHEMA_DEPRECATED_H diff --git a/tiledb/api/c_api/array_schema/array_schema_api_experimental.h b/tiledb/api/c_api/array_schema/array_schema_api_experimental.h new file mode 100644 index 000000000000..d0ca102c8651 --- /dev/null +++ b/tiledb/api/c_api/array_schema/array_schema_api_experimental.h @@ -0,0 +1,156 @@ +/** + * @file tiledb/api/c_api/array_schema/array_schema_api_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the array schema section of the experimental TileDB C API. + */ + +#ifndef TILEDB_CAPI_ARRAY_SCHEMA_EXPERIMENTAL_H +#define TILEDB_CAPI_ARRAY_SCHEMA_EXPERIMENTAL_H + +#include "../api_external_common.h" +#include "array_schema_api_external.h" + +#include "tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h" +#include "tiledb/api/c_api/enumeration/enumeration_api_experimental.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Gets timestamp range in an array schema + * + * **Example:** + * + * @code{.c} + * uint64_t timestamp_lo = 0; + * uint64_t timestamp_hi = 0; + * tiledb_array_schema_timestamp_range( + * ctx, array_schema, ×tamp_lo, ×tamp_hi); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema object. + * @param[out] lo The lower bound of timestamp range. + * @param[out] hi The upper bound of timestamp range. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_timestamp_range( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + uint64_t* lo, + uint64_t* hi) TILEDB_NOEXCEPT; + +/** + * Adds an enumeration to an array schema. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* enumeration; + * tiledb_enumeration_alloc( + * ctx, + * "enumeration_name", + * TILEDB_INT64, + * 1, + * FALSE, + * data, + * data_size, + * nullptr, + * 0, + * &enumeration); + * tiledb_array_schema_add_enumeration(ctx, array_schema, enumeration); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] enumeration The enumeration to add with the attribute + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; + +/** + * Sets the current domain on the array schema. + * + * @pre The schema is sparse. current_domain is not yet supported on dense + * arrays. + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_current_domain_create(ctx, ¤t_domain); + * tiledb_array_schema_set_current_domain(ctx, array_schema, current_domain); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] current_domain The current domain to set on the schema + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_set_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t* current_domain) TILEDB_NOEXCEPT; + +/** + * Gets the current domain set on the array schema or + * creates an empty current domain if none was set. + * It is the responsability of the caller to free the resources associated + * with the current domain when the handle isn't needed anymore. + * + * @pre The schema is sparse. current_domain is not yet supported on dense + * arrays. + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_array_schema_get_current_domain(ctx, array_schema, ¤t_domain); + * tiledb_current_domain_free(¤t_domain); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] current_domain The current domain set on the schema + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t** current_domain) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_ARRAY_SCHEMA_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/array_schema/array_schema_api_external.h b/tiledb/api/c_api/array_schema/array_schema_api_external.h new file mode 100644 index 000000000000..703c25a772fe --- /dev/null +++ b/tiledb/api/c_api/array_schema/array_schema_api_external.h @@ -0,0 +1,684 @@ +/** + * @file tiledb/api/c_api/array_schema/array_schema_api_external.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the ArraySchema C API for TileDB. + */ + +#ifndef TILEDB_CAPI_ARRAY_SCHEMA_EXTERNAL_H +#define TILEDB_CAPI_ARRAY_SCHEMA_EXTERNAL_H + +#include "../api_external_common.h" +#include "../attribute/attribute_api_external.h" +#include "../context/context_api_external.h" +#include "../domain/domain_api_external.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** C API carrier for a TileDB array schema */ +typedef struct tiledb_array_schema_handle_t tiledb_array_schema_t; + +/** Array type. */ +typedef enum { +/** Helper macro for defining array type enums. */ +#define TILEDB_ARRAY_TYPE_ENUM(id) TILEDB_##id +#include "tiledb/api/c_api/array_schema/array_type_enum.h" +#undef TILEDB_ARRAY_TYPE_ENUM +} tiledb_array_type_t; + +/** Tile or cell layout. */ +typedef enum { +/** Helper macro for defining layout type enums. */ +#define TILEDB_LAYOUT_ENUM(id) TILEDB_##id +#include "tiledb/api/c_api/array_schema/layout_enum.h" +#undef TILEDB_LAYOUT_ENUM +} tiledb_layout_t; + +/** + * Returns a string representation of the given array type. + * + * @param[in] array_type Array type + * @param[out] str A constant string representation of the array type + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_type_to_str( + tiledb_array_type_t array_type, const char** str) TILEDB_NOEXCEPT; + +/** + * Parses an array type from the given string. + * + * @param[in] str String representation to parse + * @param[out] array_type The parsed array type + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_type_from_str( + const char* str, tiledb_array_type_t* array_type) TILEDB_NOEXCEPT; + +/** + * Returns a string representation of the given layout. + * + * @param[in] layout Layout + * @param[out] str A constant string representation of the layout + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t +tiledb_layout_to_str(tiledb_layout_t layout, const char** str) TILEDB_NOEXCEPT; + +/** + * Parses a layout from the given string. + * + * @param[in] str String representation to parse + * @param[out] layout The parsed layout + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_layout_from_str( + const char* str, tiledb_layout_t* layout) TILEDB_NOEXCEPT; + +/** + * Creates a TileDB array schema object. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_t* array_schema; + * tiledb_array_schema_alloc(ctx, TILEDB_DENSE, &array_schema); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_type The array type. + * @param[out] array_schema The TileDB array schema to be created. + * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_alloc( + tiledb_ctx_t* ctx, + tiledb_array_type_t array_type, + tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; + +/** + * Destroys an array schema, freeing associated memory. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_free(&array_schema); + * @endcode + * + * @param[in] array_schema The array schema to be destroyed. + */ +TILEDB_EXPORT void tiledb_array_schema_free( + tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; + +/** + * Adds an attribute to an array schema. + * + * **Example:** + * + * @code{.c} + * tiledb_attribute_t* attr; + * tiledb_attribute_alloc(ctx, "my_attr", TILEDB_INT32, &attr); + * tiledb_array_schema_add_attribute(ctx, array_schema, attr); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] attr The attribute to be added. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_add_attribute( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_attribute_t* attr) TILEDB_NOEXCEPT; + +/** + * Sets whether the array can allow coordinate duplicates or not. + * Applicable only to sparse arrays (it errors out if set to `1` for dense + * arrays). + * + * **Example:** + * + * @code{.c} + * int allows_dups = 1; + * tiledb_array_schema_set_allows_dups(ctx, array_schema, allows_dups); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] allows_dups Whether or not the array allows coordinate duplicates. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_set_allows_dups( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + int allows_dups) TILEDB_NOEXCEPT; + +/** + * Gets whether the array can allow coordinate duplicates or not. + * It should always be `0` for dense arrays. + * + * **Example:** + * + * @code{.c} + * int allows_dups; + * tiledb_array_schema_get_allows_dups(ctx, array_schema, &allows_dups); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] allows_dups Whether or not the array allows coordinate + * duplicates. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_allows_dups( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + int* allows_dups) TILEDB_NOEXCEPT; + +/** + * Returns the array schema version. + * + * **Example:** + * + * @code{.c} + * uint32_t version; + * tiledb_array_schema_get_version(ctx, array_schema, &version); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] version The version. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_version( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + uint32_t* version) TILEDB_NOEXCEPT; + +/** + * Sets a domain for the array schema. + * + * @pre The domain has at least one dimension set. + * + * **Example:** + * + * @code{.c} + * tiledb_domain_t* domain; + * tiledb_domain_alloc(ctx, &domain); + * // -- Add dimensions to the domain here -- // + * tiledb_array_schema_set_domain(ctx, array_schema, domain); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] domain The domain to be set. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_set_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_domain_t* domain) TILEDB_NOEXCEPT; + +/** + * Sets the tile capacity. Applies to sparse arrays only. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_set_capacity(ctx, array_schema, 10000); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] capacity The capacity of a sparse data tile. Note that + * sparse data tiles exist in sparse fragments, which can be created + * in sparse arrays only. For more details, + * see [tutorials/tiling-sparse.html](tutorials/tiling-sparse.html). + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_set_capacity( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + uint64_t capacity) TILEDB_NOEXCEPT; + +/** + * Sets the cell order. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_ROW_MAJOR); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] cell_order The cell order to be set. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_set_cell_order( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_layout_t cell_order) TILEDB_NOEXCEPT; + +/** + * Sets the tile order. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_set_tile_order(ctx, array_schema, TILEDB_COL_MAJOR); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] tile_order The tile order to be set. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_set_tile_order( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_layout_t tile_order) TILEDB_NOEXCEPT; + +/** + * Sets the filter list to use for the coordinates. + * + * **Example:** + * + * @code{.c} + * tiledb_filter_list_t* filter_list; + * tiledb_filter_list_alloc(ctx, &filter_list); + * tiledb_filter_list_add_filter(ctx, filter_list, filter); + * tiledb_array_schema_set_coords_filter_list(ctx, array_schema, filter_list); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] filter_list The filter list to be set. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_set_coords_filter_list( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t* filter_list) TILEDB_NOEXCEPT; + +/** + * Sets the filter list to use for the offsets of variable-sized attribute + * values. + * + * **Example:** + * + * @code{.c} + * tiledb_filter_list_t* filter_list; + * tiledb_filter_list_alloc(ctx, &filter_list); + * tiledb_filter_list_add_filter(ctx, filter_list, filter); + * tiledb_array_schema_set_offsets_filter_list(ctx, array_schema, filter_list); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] filter_list The filter list to be set. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_set_offsets_filter_list( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t* filter_list) TILEDB_NOEXCEPT; + +/** + * Sets the filter list to use for the validity array of nullable attribute + * values. + * + * **Example:** + * + * @code{.c} + * tiledb_filter_list_t* filter_list; + * tiledb_filter_list_alloc(ctx, &filter_list); + * tiledb_filter_list_add_filter(ctx, filter_list, filter); + * tiledb_array_schema_set_validity_filter_list(ctx, array_schema, filter_list); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] filter_list The filter list to be set. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_set_validity_filter_list( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t* filter_list) TILEDB_NOEXCEPT; + +/** + * Checks the correctness of the array schema. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_check(ctx, array_schema); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @return `TILEDB_OK` if the array schema is correct and `TILEDB_ERR` upon any + * error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_check( + tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema) TILEDB_NOEXCEPT; + +/** + * Retrieves the array type. + * + * **Example:** + * + * @code{.c} + * tiledb_array_schema_t* array_schema; + * tiledb_array_schema_load(ctx, "s3://tiledb_bucket/my_array", array_schema); + * tiledb_array_type_t* array_type; + * tiledb_array_schema_get_array_type(ctx, array_schema, &array_type); + * // Make sure to free the array schema in the end + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] array_type The array type to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_array_type( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_array_type_t* array_type) TILEDB_NOEXCEPT; + +/** + * Retrieves the capacity. + * + * **Example:** + * + * @code{.c} + * uint64_t capacity; + * tiledb_array_schema_get_capacity(ctx, array_schema, &capacity); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] capacity The capacity to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_capacity( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + uint64_t* capacity) TILEDB_NOEXCEPT; + +/** + * Retrieves the cell order. + * + * **Example:** + * + * @code{.c} + * tiledb_layout_t cell_order; + * tiledb_array_schema_get_cell_order(ctx, array_schema, &cell_order); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] cell_order The cell order to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_cell_order( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_layout_t* cell_order) TILEDB_NOEXCEPT; + +/** + * Retrieves the filter list used for the coordinates. + * + * **Example:** + * + * @code{.c} + * tiledb_filter_list_t* filter_list; + * tiledb_array_schema_get_coords_filter_list(ctx, array_schema, &filter_list); + * tiledb_filter_list_free(ctx, &filter_list); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] filter_list The filter list to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_coords_filter_list( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t** filter_list) TILEDB_NOEXCEPT; + +/** + * Retrieves the filter list used for the offsets. + * + * **Example:** + * + * @code{.c} + * tiledb_filter_list_t* filter_list; + * tiledb_array_schema_get_offsets_filter_list(ctx, array_schema, &filter_list); + * tiledb_filter_list_free(ctx, &filter_list); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] filter_list The filter list to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_offsets_filter_list( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t** filter_list) TILEDB_NOEXCEPT; + +/** + * Retrieves the filter list used for validity maps. + * + * **Example:** + * + * @code{.c} + * tiledb_filter_list_t* filter_list; + * tiledb_array_schema_get_validity_filter_list( + * ctx, array_schema, &filter_list); + * tiledb_filter_list_free(ctx, &filter_list); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] filter_list The filter list to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_validity_filter_list( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_filter_list_t** filter_list) TILEDB_NOEXCEPT; + +/** + * Retrieves the array domain. + * + * **Example:** + * + * @code{.c} + * tiledb_domain_t* domain; + * tiledb_array_schema_get_domain(ctx, array_schema, &domain); + * tiledb_domain_free(&domain); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] domain The array domain to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_domain( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_domain_t** domain) TILEDB_NOEXCEPT; + +/** + * Retrieves the tile order. + * + * **Example:** + * + * @code{.c} + * tiledb_layout_t tile_order; + * tiledb_array_schema_get_tile_order(ctx, array_schema, &tile_order); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] tile_order The tile order to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_tile_order( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_layout_t* tile_order) TILEDB_NOEXCEPT; + +/** + * Retrieves the number of array attributes. + * + * **Example:** + * + * @code{.c} + * uint32_t attr_num; + * tiledb_array_schema_get_attribute_num(ctx, array_schema, &attr_num); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] attribute_num The number of attributes to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_attribute_num( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + uint32_t* attribute_num) TILEDB_NOEXCEPT; + +/** + * Retrieves an attribute given its index. + * + * Attributes are ordered the same way they were defined + * when constructing the array schema. + * + * **Example:** + * + * The following retrieves the first attribute in the schema. + * + * @code{.c} + * tiledb_attribute_t* attr; + * tiledb_array_schema_get_attribute_from_index(ctx, array_schema, 0, &attr); + * tiledb_attribute_free(&attr); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] index The index of the attribute to retrieve. + * @param[out] attr The attribute object to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_attribute_from_index( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + uint32_t index, + tiledb_attribute_t** attr) TILEDB_NOEXCEPT; + +/** + * Retrieves an attribute given its name (key). + * + * **Example:** + * + * The following retrieves the attribute named "a" in the schema. + * + * @code{.c} + * tiledb_attribute_t* attr; + * tiledb_array_schema_get_attribute_from_name(ctx, array_schema, "a", &attr); + * tiledb_attribute_free(&attr); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] name The name (key) of the attribute to retrieve. + * @param[out] attr The attribute object to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_get_attribute_from_name( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + const char* name, + tiledb_attribute_t** attr) TILEDB_NOEXCEPT; + +/** + * Checks whether the array schema has an attribute of the given name. + * + * **Example:** + * + * @code{.c} + * int32_t has_attr; + * tiledb_array_schema_has_attribute(ctx, array_schema, "attr_0", &has_attr); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[in] name The name of the attribute to check for. + * @param[out] has_attr Set to `1` if the array schema has an attribute of the + * given name, else `0`. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_has_attribute( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + const char* name, + int32_t* has_attr) TILEDB_NOEXCEPT; + +/** + * Dumps the array schema in ASCII format in the selected string output. + * + * The output string handle must be freed by the user after use. + * + * **Example:** + * + * @code{.c} + * tiledb_string_t* tdb_string; + * tiledb_array_schema_dump_str(ctx, array_schema, &tdb_string); + * // Use the string + * tiledb_string_free(&tdb_string); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] array_schema The array schema. + * @param[out] out The output string. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_dump_str( + tiledb_ctx_t* ctx, + const tiledb_array_schema_t* array_schema, + tiledb_string_t** out) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_ARRAY_SCHEMA_EXTERNAL_H diff --git a/tiledb/api/c_api/array_schema/array_schema_api_internal.h b/tiledb/api/c_api/array_schema/array_schema_api_internal.h new file mode 100644 index 000000000000..89ad84399d85 --- /dev/null +++ b/tiledb/api/c_api/array_schema/array_schema_api_internal.h @@ -0,0 +1,259 @@ +/** + * @file tiledb/api/c_api/array_schema/array_schema_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the internals of the array_schema section of the C API. + */ + +#ifndef TILEDB_CAPI_ARRAY_SCHEMA_INTERNAL_H +#define TILEDB_CAPI_ARRAY_SCHEMA_INTERNAL_H + +#include "array_schema_api_external.h" + +#include "tiledb/api/c_api_support/handle/handle.h" +#include "tiledb/common/common.h" +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/layout.h" + +using namespace tiledb::sm; + +/** Handle `struct` for API ArraySchema objects. */ +struct tiledb_array_schema_handle_t + : public tiledb::api::CAPIHandle { + /** Type name */ + static constexpr std::string_view object_type_name{"array_schema"}; + + private: + using array_schema_type = shared_ptr; + array_schema_type array_schema_; + + public: + template + explicit tiledb_array_schema_handle_t(Args... args) + : array_schema_{ + make_shared(HERE(), std::forward(args)...)} { + } + + explicit tiledb_array_schema_handle_t(const ArraySchema& array_schema) + : array_schema_{make_shared(HERE(), array_schema)} { + } + + /** + * Constructor from `shared_ptr` copies the shared pointer. + */ + explicit tiledb_array_schema_handle_t(const array_schema_type& x) + : array_schema_(x) { + } + + array_schema_type array_schema() const { + return array_schema_; + } + + Status add_attribute( + shared_ptr attr, bool check_special = true) { + return array_schema_->add_attribute(attr, check_special); + } + + void add_dimension_label( + ArraySchema::dimension_size_type dim_id, + const std::string& name, + DataOrder label_order, + Datatype label_type, + bool check_name = true) { + array_schema_->add_dimension_label( + dim_id, name, label_order, label_type, check_name); + } + + void add_enumeration(shared_ptr enmr) { + return array_schema_->add_enumeration(enmr); + } + + bool allows_dups() const { + return array_schema_->allows_dups(); + } + + ArrayType array_type() const { + return array_schema_->array_type(); + } + + const URI& array_uri() const { + return array_schema_->array_uri(); + } + + ArraySchema::attribute_size_type attribute_num() const { + return array_schema_->attribute_num(); + } + + uint64_t capacity() const { + return array_schema_->capacity(); + } + + Layout cell_order() const { + return array_schema_->cell_order(); + } + + const FilterPipeline& cell_validity_filters() const { + return array_schema_->cell_validity_filters(); + } + + const FilterPipeline& cell_var_offsets_filters() const { + return array_schema_->cell_var_offsets_filters(); + } + + void check(const Config& cfg) const { + array_schema_->check(cfg); + } + + const FilterPipeline& coords_filters() const { + return array_schema_->coords_filters(); + } + + const DimensionLabel& dimension_label( + ArraySchema::dimension_label_size_type i) const { + return array_schema_->dimension_label(i); + } + + const DimensionLabel& dimension_label(const std::string& name) const { + return array_schema_->dimension_label(name); + } + + const Dimension* dimension_ptr(ArraySchema::dimension_size_type i) const { + return array_schema_->dimension_ptr(i); + } + + const Dimension* dimension_ptr(const std::string& name) const { + return array_schema_->dimension_ptr(name); + } + + ArraySchema::dimension_label_size_type dim_label_num() const { + return array_schema_->dim_label_num(); + } + + shared_ptr get_current_domain() const { + return array_schema_->get_current_domain(); + } + + Status has_attribute(const std::string& name, bool* has_attr) const { + return array_schema_->has_attribute(name, has_attr); + } + + bool is_dim_label(const std::string& name) const { + return array_schema_->is_dim_label(name); + } + + Status set_allows_dups(bool allows_dups) { + return array_schema_->set_allows_dups(allows_dups); + } + + void set_capacity(uint64_t capacity) { + array_schema_->set_capacity(capacity); + } + + void set_current_domain(shared_ptr current_domain) { + array_schema_->set_current_domain(current_domain); + } + + void set_dimension_label_filter_pipeline( + const std::string& label_name, const FilterPipeline& pipeline) { + array_schema_->set_dimension_label_filter_pipeline(label_name, pipeline); + } + + void set_dimension_label_tile_extent( + const std::string& label_name, + const Datatype type, + const void* tile_extent) { + array_schema_->set_dimension_label_tile_extent( + label_name, type, tile_extent); + } + + Status set_domain(shared_ptr domain) { + return array_schema_->set_domain(domain); + } + + Status set_cell_order(Layout cell_order) { + return array_schema_->set_cell_order(cell_order); + } + + Status set_cell_validity_filter_pipeline(const FilterPipeline& pipeline) { + return array_schema_->set_cell_validity_filter_pipeline(pipeline); + } + + Status set_cell_var_offsets_filter_pipeline(const FilterPipeline& pipeline) { + return array_schema_->set_cell_var_offsets_filter_pipeline(pipeline); + } + + Status set_coords_filter_pipeline(const FilterPipeline& pipeline) { + return array_schema_->set_coords_filter_pipeline(pipeline); + } + + Status set_tile_order(Layout tile_order) { + return array_schema_->set_tile_order(tile_order); + } + + shared_ptr shared_attribute( + ArraySchema::attribute_size_type id) const { + return array_schema_->shared_attribute(id); + } + + shared_ptr shared_attribute(const std::string& name) const { + return array_schema_->shared_attribute(name); + } + + shared_ptr shared_domain() const { + return array_schema_->shared_domain(); + } + + Layout tile_order() const { + return array_schema_->tile_order(); + } + + std::pair timestamp_range() const { + return array_schema_->timestamp_range(); + } + + format_version_t version() const { + return array_schema_->version(); + } +}; + +namespace tiledb::api { + +/** + * Returns after successfully validating an array schema. + * + * @param array_schema Possibly-valid pointer to an array schema + */ +inline void ensure_array_schema_is_valid( + const tiledb_array_schema_t* array_schema) { + ensure_handle_is_valid(array_schema); +} + +} // namespace tiledb::api + +#endif // TILEDB_CAPI_ARRAY_SCHEMA_INTERNAL_H diff --git a/tiledb/api/c_api/array_schema/array_type_enum.h b/tiledb/api/c_api/array_schema/array_type_enum.h new file mode 100644 index 000000000000..0a53167517cd --- /dev/null +++ b/tiledb/api/c_api/array_schema/array_type_enum.h @@ -0,0 +1,41 @@ +/* + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/** + * NOTE: The values of these enums are serialized to the array schema and/or + * fragment metadata. Therefore, the values below should never change, + * otherwise backwards compatibility breaks. + */ + +// clang-format is disabled on the first enum so that we can manually indent it +// properly. + +// clang-format off +#ifdef TILEDB_ARRAY_TYPE_ENUM + /** Dense array */ + TILEDB_ARRAY_TYPE_ENUM(DENSE) = 0, + /** Sparse array */ + TILEDB_ARRAY_TYPE_ENUM(SPARSE) = 1, +#endif + // clang-format on diff --git a/tiledb/api/c_api/array_schema/layout_enum.h b/tiledb/api/c_api/array_schema/layout_enum.h new file mode 100644 index 000000000000..c2b24a8c44a4 --- /dev/null +++ b/tiledb/api/c_api/array_schema/layout_enum.h @@ -0,0 +1,47 @@ +/* + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/** + * NOTE: The values of these enums are serialized to the array schema and/or + * fragment metadata. Therefore, the values below should never change, + * otherwise backwards compatibility breaks. + */ + +// clang-format is disabled on the first enum so that we can manually indent it +// properly. + +// clang-format off +#ifdef TILEDB_LAYOUT_ENUM + /** Row-major layout */ + TILEDB_LAYOUT_ENUM(ROW_MAJOR) = 0, + /** Column-major layout */ + TILEDB_LAYOUT_ENUM(COL_MAJOR) = 1, + /** Global-order layout */ + TILEDB_LAYOUT_ENUM(GLOBAL_ORDER) = 2, + /** Unordered layout */ + TILEDB_LAYOUT_ENUM(UNORDERED) = 3, + /** Hilbert layout */ + TILEDB_LAYOUT_ENUM(HILBERT) = 4, +#endif + // clang-format on diff --git a/tiledb/api/c_api/array_schema/test/CMakeLists.txt b/tiledb/api/c_api/array_schema/test/CMakeLists.txt new file mode 100644 index 000000000000..a61b624205b1 --- /dev/null +++ b/tiledb/api/c_api/array_schema/test/CMakeLists.txt @@ -0,0 +1,37 @@ +# +# tiledb/api/c_api/array_schema/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_array_schema) + this_target_sources(unit_capi_array_schema.cc) + this_target_object_libraries( + capi_array_schema_stub + capi_attribute_stub + capi_current_domain + capi_domain_stub + ) +conclude(unit_test) diff --git a/tiledb/api/c_api/array_schema/test/compile_capi_array_schema_stub_main.cc b/tiledb/api/c_api/array_schema/test/compile_capi_array_schema_stub_main.cc new file mode 100644 index 000000000000..588594caa46c --- /dev/null +++ b/tiledb/api/c_api/array_schema/test/compile_capi_array_schema_stub_main.cc @@ -0,0 +1,35 @@ +/** + * @file + * tiledb/api/c_api/array_schema/test/compile_capi_array_schema_stub_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../array_schema_api_external.h" + +int main() { + tiledb_array_schema_add_attribute(nullptr, nullptr, nullptr); + return 0; +} diff --git a/tiledb/api/c_api/array_schema/test/unit_capi_array_schema.cc b/tiledb/api/c_api/array_schema/test/unit_capi_array_schema.cc new file mode 100644 index 000000000000..8e86cc4c5e6e --- /dev/null +++ b/tiledb/api/c_api/array_schema/test/unit_capi_array_schema.cc @@ -0,0 +1,855 @@ +/** + * @file tiledb/api/c_api/array_schema/test/unit_capi_array_schema.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * Validates the arguments for the ArraySchema C API. + */ + +#define CATCH_CONFIG_MAIN +#include +#include "../../../c_api_test_support/testsupport_capi_array_schema.h" +#include "../../../c_api_test_support/testsupport_capi_context.h" +#include "../array_schema_api_experimental.h" +#include "../array_schema_api_external.h" +#include "../array_schema_api_internal.h" + +using namespace tiledb::api::test_support; + +TEST_CASE( + "C API: tiledb_array_schema_alloc argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_context ctx{}; + tiledb_array_schema_handle_t* schema{}; + SECTION("success") { + rc = tiledb_array_schema_alloc(ctx.context, TILEDB_DENSE, &schema); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + tiledb_array_schema_free(&schema); + } + SECTION("null context") { + rc = tiledb_array_schema_alloc(nullptr, TILEDB_DENSE, &schema); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + /* + * SECTION("invalid array_type"): All values of `array_type` may be valid + * in certain circumstances. Not checked in the API code and not tested here. + */ + SECTION("null schema") { + rc = tiledb_array_schema_alloc(ctx.context, TILEDB_DENSE, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_free argument validation", + "[capi][array_schema]") { + ordinary_context ctx{}; + tiledb_array_schema_handle_t* schema{}; + auto rc = tiledb_array_schema_alloc(ctx.context, TILEDB_DENSE, &schema); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + SECTION("success") { + REQUIRE_NOTHROW(tiledb_array_schema_free(&schema)); + CHECK(schema == nullptr); + } + SECTION("null schema") { + /* + * `tiledb_array_schema_free` is a void function, otherwise we would check + * for an error. + */ + REQUIRE_NOTHROW(tiledb_array_schema_free(nullptr)); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_add_attribute argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_attribute_t* attr{}; + rc = tiledb_attribute_alloc(x.ctx(), "a", TILEDB_INT32, &attr); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + SECTION("success") { + rc = tiledb_array_schema_add_attribute(x.ctx(), x.schema, attr); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_add_attribute(nullptr, x.schema, attr); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_add_attribute(x.ctx(), nullptr, attr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null attribute") { + rc = tiledb_array_schema_add_attribute(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + REQUIRE_NOTHROW(tiledb_attribute_free(&attr)); + CHECK(attr == nullptr); +} + +TEST_CASE( + "C API: tiledb_array_schema_set_allows_dups argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{TILEDB_DENSE}; + SECTION("success") { + rc = tiledb_array_schema_set_allows_dups(x.ctx(), x.schema, 0); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_set_allows_dups(nullptr, x.schema, 0); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_set_allows_dups(x.ctx(), nullptr, 0); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("invalid allows_dups") { + /* + * This API is applicable _only_ to sparse arrays. + * As such, any non-zero value set on a dense array is considered invalid. + */ + rc = tiledb_array_schema_set_allows_dups(x.ctx(), x.schema, 1); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_allows_dups argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + int allows_dups; + SECTION("success") { + rc = tiledb_array_schema_get_allows_dups(x.ctx(), x.schema, &allows_dups); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_allows_dups(nullptr, x.schema, &allows_dups); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_allows_dups(x.ctx(), nullptr, &allows_dups); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null allows_dups") { + rc = tiledb_array_schema_get_allows_dups(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_version argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + uint32_t version; + SECTION("success") { + rc = tiledb_array_schema_get_version(x.ctx(), x.schema, &version); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_version(nullptr, x.schema, &version); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_version(x.ctx(), nullptr, &version); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null version") { + rc = tiledb_array_schema_get_version(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_set_domain argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_domain_t* domain{}; + rc = tiledb_domain_alloc(x.ctx(), &domain); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + ordinary_dimension_d1 dim; + rc = tiledb_domain_add_dimension(x.ctx(), domain, dim.dimension); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + SECTION("success") { + rc = tiledb_array_schema_set_domain(x.ctx(), x.schema, domain); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_set_domain(nullptr, x.schema, domain); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_set_domain(x.ctx(), nullptr, domain); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null domain") { + rc = tiledb_array_schema_set_domain(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + REQUIRE_NOTHROW(tiledb_domain_free(&domain)); + CHECK(domain == nullptr); +} + +TEST_CASE( + "C API: tiledb_array_schema_set_capacity argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + SECTION("success") { + rc = tiledb_array_schema_set_capacity(x.ctx(), x.schema, 1); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_set_capacity(nullptr, x.schema, 1); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_set_capacity(x.ctx(), nullptr, 1); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("invalid capacity") { + /* The capacity may not be zero. */ + rc = tiledb_array_schema_set_capacity(x.ctx(), x.schema, 0); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_set_cell_order argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_layout_t layout = TILEDB_ROW_MAJOR; + SECTION("success") { + rc = tiledb_array_schema_set_cell_order(x.ctx(), x.schema, layout); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_set_cell_order(nullptr, x.schema, layout); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_set_cell_order(x.ctx(), nullptr, layout); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("invalid layout") { + /* A cell order of TILEDB_UNORDERED is not yet supported. */ + rc = + tiledb_array_schema_set_cell_order(x.ctx(), x.schema, TILEDB_UNORDERED); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_set_tile_order argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{TILEDB_DENSE}; + tiledb_layout_t layout = TILEDB_ROW_MAJOR; + SECTION("success") { + rc = tiledb_array_schema_set_tile_order(x.ctx(), x.schema, layout); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_set_tile_order(nullptr, x.schema, layout); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_set_tile_order(x.ctx(), nullptr, layout); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("invalid layout") { + /* ordinary_array_schema is dense, which disallows unordered layouts. */ + rc = + tiledb_array_schema_set_tile_order(x.ctx(), x.schema, TILEDB_UNORDERED); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_timestamp_range argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + uint64_t lo; + uint64_t hi; + SECTION("success") { + rc = tiledb_array_schema_timestamp_range(x.ctx(), x.schema, &lo, &hi); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_timestamp_range(nullptr, x.schema, &lo, &hi); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_timestamp_range(x.ctx(), nullptr, &lo, &hi); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null lo") { + rc = tiledb_array_schema_timestamp_range(x.ctx(), x.schema, nullptr, &hi); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null hi") { + rc = tiledb_array_schema_timestamp_range(x.ctx(), x.schema, &lo, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_add_enumeration argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_enumeration_t* enumeration; + int32_t values[5] = {1, 2, 3, 4, 5}; + rc = tiledb_enumeration_alloc( + x.ctx(), + "enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + SECTION("success") { + rc = tiledb_array_schema_add_enumeration(x.ctx(), x.schema, enumeration); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_add_enumeration(nullptr, x.schema, enumeration); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_add_enumeration(x.ctx(), nullptr, enumeration); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null enumeration") { + rc = tiledb_array_schema_add_enumeration(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + REQUIRE_NOTHROW(tiledb_enumeration_free(&enumeration)); + CHECK(enumeration == nullptr); +} + +TEST_CASE( + "C API: tiledb_array_schema_set_coords_filter_list argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_filter_list_t* fl; + rc = tiledb_filter_list_alloc(x.ctx(), &fl); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + SECTION("success") { + rc = tiledb_array_schema_set_coords_filter_list(x.ctx(), x.schema, fl); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_set_coords_filter_list(nullptr, x.schema, fl); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_set_coords_filter_list(x.ctx(), nullptr, fl); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null filter_list") { + rc = tiledb_array_schema_set_coords_filter_list(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + REQUIRE_NOTHROW(tiledb_filter_list_free(&fl)); + CHECK(fl == nullptr); +} + +TEST_CASE( + "C API: tiledb_array_schema_set_offsets_filter_list argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_filter_list_t* fl; + rc = tiledb_filter_list_alloc(x.ctx(), &fl); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + SECTION("success") { + rc = tiledb_array_schema_set_offsets_filter_list(x.ctx(), x.schema, fl); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_set_offsets_filter_list(nullptr, x.schema, fl); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_set_offsets_filter_list(x.ctx(), nullptr, fl); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null filter_list") { + rc = + tiledb_array_schema_set_offsets_filter_list(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + REQUIRE_NOTHROW(tiledb_filter_list_free(&fl)); + CHECK(fl == nullptr); +} + +TEST_CASE( + "C API: tiledb_array_schema_set_validity_filter_list argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_filter_list_t* fl; + rc = tiledb_filter_list_alloc(x.ctx(), &fl); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + SECTION("success") { + rc = tiledb_array_schema_set_validity_filter_list(x.ctx(), x.schema, fl); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_set_validity_filter_list(nullptr, x.schema, fl); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_set_validity_filter_list(x.ctx(), nullptr, fl); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null filter_list") { + rc = tiledb_array_schema_set_validity_filter_list( + x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + REQUIRE_NOTHROW(tiledb_filter_list_free(&fl)); + CHECK(fl == nullptr); +} + +TEST_CASE( + "C API: tiledb_array_schema_check argument validation", + "[capi][array_schema]") { + /* + * No "success" section here; too much overhead to set up. + */ + capi_return_t rc; + ordinary_array_schema x{}; + SECTION("null context") { + rc = tiledb_array_schema_check(nullptr, x.schema); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_check(x.ctx(), nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_array_type argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_array_type_t array_type; + SECTION("success") { + rc = tiledb_array_schema_get_array_type(x.ctx(), x.schema, &array_type); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_array_type(nullptr, x.schema, &array_type); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_array_type(x.ctx(), nullptr, &array_type); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null array_type") { + rc = tiledb_array_schema_get_array_type(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_capacity argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + uint64_t capacity; + SECTION("success") { + rc = tiledb_array_schema_get_capacity(x.ctx(), x.schema, &capacity); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_capacity(nullptr, x.schema, &capacity); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_capacity(x.ctx(), nullptr, &capacity); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null capacity") { + rc = tiledb_array_schema_get_capacity(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_cell_order argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_layout_t cell_order; + SECTION("success") { + rc = tiledb_array_schema_get_cell_order(x.ctx(), x.schema, &cell_order); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_cell_order(nullptr, x.schema, &cell_order); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_cell_order(x.ctx(), nullptr, &cell_order); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null cell_order") { + rc = tiledb_array_schema_get_cell_order(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_coords_filter_list argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_filter_list_t* fl; + SECTION("success") { + rc = tiledb_array_schema_get_coords_filter_list(x.ctx(), x.schema, &fl); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_coords_filter_list(nullptr, x.schema, &fl); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_coords_filter_list(x.ctx(), nullptr, &fl); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null filter_list") { + rc = tiledb_array_schema_get_coords_filter_list(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_offsets_filter_list argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_filter_list_t* fl; + SECTION("success") { + rc = tiledb_array_schema_get_offsets_filter_list(x.ctx(), x.schema, &fl); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_offsets_filter_list(nullptr, x.schema, &fl); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_offsets_filter_list(x.ctx(), nullptr, &fl); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null filter_list") { + rc = + tiledb_array_schema_get_offsets_filter_list(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_validity_filter_list argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_filter_list_t* fl; + SECTION("success") { + rc = tiledb_array_schema_get_validity_filter_list(x.ctx(), x.schema, &fl); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_validity_filter_list(nullptr, x.schema, &fl); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_validity_filter_list(x.ctx(), nullptr, &fl); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null filter_list") { + rc = tiledb_array_schema_get_validity_filter_list( + x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_domain argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_domain_t* domain; + SECTION("success") { + rc = tiledb_array_schema_get_domain(x.ctx(), x.schema, &domain); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_domain(nullptr, x.schema, &domain); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_domain(x.ctx(), nullptr, &domain); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null domain") { + rc = tiledb_array_schema_get_domain(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_tile_order argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_layout_t tile_order; + SECTION("success") { + rc = tiledb_array_schema_get_tile_order(x.ctx(), x.schema, &tile_order); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_tile_order(nullptr, x.schema, &tile_order); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_tile_order(x.ctx(), nullptr, &tile_order); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null tile_order") { + rc = tiledb_array_schema_get_tile_order(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_attribute_num argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + uint32_t attr_num; + SECTION("success") { + rc = tiledb_array_schema_get_attribute_num(x.ctx(), x.schema, &attr_num); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_attribute_num(nullptr, x.schema, &attr_num); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_attribute_num(x.ctx(), nullptr, &attr_num); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null attribute_num") { + rc = tiledb_array_schema_get_attribute_num(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_dump_str argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_string_t* out; + /* + * No "success" section here; omitted to avoid log noise. + */ + SECTION("null context") { + rc = tiledb_array_schema_dump_str(nullptr, x.schema, &out); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_dump_str(x.ctx(), nullptr, &out); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + /* + * SECTION("null file pointer"): `nullptr` is allowed; it's mapped to `stdout` + */ +} + +TEST_CASE( + "C API: tiledb_array_schema_get_attribute_from_index argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema_with_attr x{}; + tiledb_attribute_t* attr; + SECTION("success") { + rc = tiledb_array_schema_get_attribute_from_index( + x.ctx(), x.schema, 0, &attr); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_attribute_from_index( + nullptr, x.schema, 0, &attr); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_attribute_from_index( + x.ctx(), nullptr, 0, &attr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("invalid index") { + rc = tiledb_array_schema_get_attribute_from_index( + x.ctx(), x.schema, 1, &attr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null attribute") { + rc = tiledb_array_schema_get_attribute_from_index( + x.ctx(), x.schema, 0, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_get_attribute_from_name argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema_with_attr x{}; + tiledb_attribute_t* attr; + SECTION("success") { + rc = tiledb_array_schema_get_attribute_from_name( + x.ctx(), x.schema, "a", &attr); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_attribute_from_name( + nullptr, x.schema, "a", &attr); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_attribute_from_name( + x.ctx(), nullptr, "a", &attr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("invalid name") { + rc = tiledb_array_schema_get_attribute_from_name( + x.ctx(), x.schema, "b", &attr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null attribute") { + rc = tiledb_array_schema_get_attribute_from_name( + x.ctx(), x.schema, "a", nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_has_attribute argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema_with_attr x{}; + int32_t has_attr; + SECTION("success") { + rc = tiledb_array_schema_has_attribute(x.ctx(), x.schema, "a", &has_attr); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + CHECK(has_attr == 1); + } + SECTION("null context") { + rc = tiledb_array_schema_has_attribute(nullptr, x.schema, "a", &has_attr); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_has_attribute(x.ctx(), nullptr, "a", &has_attr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("invalid name") { + // Note: this test is successful, but has_attr will equate to false (0). + rc = tiledb_array_schema_has_attribute(x.ctx(), x.schema, "b", &has_attr); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + CHECK(has_attr == 0); + } + SECTION("null has_attr") { + rc = tiledb_array_schema_has_attribute(x.ctx(), x.schema, "a", nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_array_schema_set_current_domain argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_current_domain_t* cd{}; + rc = tiledb_current_domain_create(x.ctx(), &cd); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + SECTION("success") { + rc = tiledb_array_schema_set_current_domain(x.ctx(), x.schema, cd); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_set_current_domain(nullptr, x.schema, cd); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_set_current_domain(x.ctx(), nullptr, cd); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null current_domain") { + rc = tiledb_array_schema_set_current_domain(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + REQUIRE_NOTHROW(tiledb_current_domain_free(&cd)); + CHECK(cd == nullptr); +} + +TEST_CASE( + "C API: tiledb_array_schema_get_current_domain argument validation", + "[capi][array_schema]") { + capi_return_t rc; + ordinary_array_schema x{}; + tiledb_current_domain_t* cd; + SECTION("success") { + rc = tiledb_array_schema_get_current_domain(x.ctx(), x.schema, &cd); + REQUIRE(tiledb_status(rc) == TILEDB_OK); + } + SECTION("null context") { + rc = tiledb_array_schema_get_current_domain(nullptr, x.schema, &cd); + REQUIRE(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null schema") { + rc = tiledb_array_schema_get_current_domain(x.ctx(), nullptr, &cd); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null current_domain") { + rc = tiledb_array_schema_get_current_domain(x.ctx(), x.schema, nullptr); + REQUIRE(tiledb_status(rc) == TILEDB_ERR); + } +} diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index b57b58a3c7b6..73893c7c86ba 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -740,13 +740,17 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * If true, array non empty domain will be loaded and sent to server together * with the open array
* **Default**: true + * - `rest.load_enumerations_on_array_open`
+ * If true, enumerations will be loaded and sent to server together with + * the open array. + * **Default**: false * - `rest.use_refactored_array_open`
* If true, the new REST routes and APIs for opening an array * will be used
* **Default**: true * - `rest.use_refactored_array_open_and_query_submit`
* If true, the new REST routes and APIs for opening an array and submitting - * a query will be used
+ * a query will be used
* **Default**: true * - `rest.curl.buffer_size`
* Set curl buffer size for REST requests
diff --git a/tiledb/api/c_api/context/CMakeLists.txt b/tiledb/api/c_api/context/CMakeLists.txt index b5a6cf61dd39..646f335e6692 100644 --- a/tiledb/api/c_api/context/CMakeLists.txt +++ b/tiledb/api/c_api/context/CMakeLists.txt @@ -35,7 +35,6 @@ gather_sources(${SOURCES}) list(APPEND OTHER_SOURCES # We need to recompile sources that depend on StorageManager to use the stub ../../../sm/storage_manager/context.cc - ../../../sm/storage_manager/context_resources.cc ) commence(object_library capi_context_stub) @@ -43,7 +42,7 @@ commence(object_library capi_context_stub) this_target_link_libraries(export) this_target_object_libraries(capi_config_stub) this_target_object_libraries(storage_manager_stub) - this_target_object_libraries(rest_client stats thread_pool vfs) + this_target_object_libraries(context_resources) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/api/c_api/current_domain/current_domain_api.cc b/tiledb/api/c_api/current_domain/current_domain_api.cc index 49b219a21c30..d28ac6829249 100644 --- a/tiledb/api/c_api/current_domain/current_domain_api.cc +++ b/tiledb/api/c_api/current_domain/current_domain_api.cc @@ -106,6 +106,7 @@ capi_return_t tiledb_current_domain_get_type( } // namespace tiledb::api +using tiledb::api::api_entry_context; using tiledb::api::api_entry_plain; using tiledb::api::api_entry_with_context; @@ -124,32 +125,36 @@ CAPI_INTERFACE(current_domain_free, tiledb_current_domain_t** current_domain) { CAPI_INTERFACE( current_domain_set_ndrectangle, + tiledb_ctx_t* ctx, tiledb_current_domain_t* current_domain, tiledb_ndrectangle_t* ndr) { - return api_entry_plain( - current_domain, ndr); + return api_entry_context( + ctx, current_domain, ndr); } CAPI_INTERFACE( current_domain_get_ndrectangle, + tiledb_ctx_t* ctx, tiledb_current_domain_t* current_domain, tiledb_ndrectangle_t** ndr) { - return api_entry_plain( - current_domain, ndr); + return api_entry_context( + ctx, current_domain, ndr); } CAPI_INTERFACE( current_domain_get_is_empty, + tiledb_ctx_t* ctx, tiledb_current_domain_t* current_domain, uint32_t* is_empty) { - return api_entry_plain( - current_domain, is_empty); + return api_entry_context( + ctx, current_domain, is_empty); } CAPI_INTERFACE( current_domain_get_type, + tiledb_ctx_t* ctx, tiledb_current_domain_t* current_domain, tiledb_current_domain_type_t* type) { - return api_entry_plain( - current_domain, type); + return api_entry_context( + ctx, current_domain, type); } diff --git a/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h b/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h index cc3db794f1ad..8d5499a2d8cc 100644 --- a/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h +++ b/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h @@ -106,17 +106,19 @@ TILEDB_EXPORT capi_return_t tiledb_current_domain_free( * range.max_size = sizeof(max); * tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim", &range); * - * tiledb_current_domain_set_ndrectangle(current_domain, ndr); + * tiledb_current_domain_set_ndrectangle(ctx, current_domain, ndr); * * tiledb_ndrectangle_free(&ndr); * tiledb_current_domain_free(¤t_domain); * @endcode * + * @param ctx A TileDB context * @param current_domain The current domain to modify * @param ndr The N-dimensional rectangle to be set * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT capi_return_t tiledb_current_domain_set_ndrectangle( + tiledb_ctx_t* ctx, tiledb_current_domain_t* current_domain, tiledb_ndrectangle_t* ndr) TILEDB_NOEXCEPT; @@ -131,14 +133,16 @@ TILEDB_EXPORT capi_return_t tiledb_current_domain_set_ndrectangle( * * @code{.c} * tiledb_ndrectangle_t *ndr; - * tiledb_current_domain_get_ndrectangle(current_domain, &ndr); + * tiledb_current_domain_get_ndrectangle(ctx, current_domain, &ndr); * @endcode * + * @param ctx A TileDB context * @param current_domain The current domain to query * @param ndr The N-dimensional rectangle of the current domain * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT capi_return_t tiledb_current_domain_get_ndrectangle( + tiledb_ctx_t* ctx, tiledb_current_domain_t* current_domain, tiledb_ndrectangle_t** ndr) TILEDB_NOEXCEPT; @@ -149,14 +153,16 @@ TILEDB_EXPORT capi_return_t tiledb_current_domain_get_ndrectangle( * * @code{.c} * uint32_t empty = 0; - * tiledb_current_domain_get_is_empty(current_domain, &empty); + * tiledb_current_domain_get_is_empty(ctx, current_domain, &empty); * @endcode * + * @param ctx A TileDB context * @param current_domain The current domain to query * @param is_empty True if nothing is set on the current domain * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT capi_return_t tiledb_current_domain_get_is_empty( + tiledb_ctx_t* ctx, tiledb_current_domain_t* current_domain, uint32_t* is_empty) TILEDB_NOEXCEPT; @@ -167,14 +173,16 @@ TILEDB_EXPORT capi_return_t tiledb_current_domain_get_is_empty( * * @code{.c} * tiledb_current_domain_type_t type; - * tiledb_current_domain_get_type(current_domain, &type); + * tiledb_current_domain_get_type(ctx, current_domain, &type); * @endcode * + * @param ctx A TileDB context * @param current_domain The current domain to query * @param type The type of representation set on the current domain * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT capi_return_t tiledb_current_domain_get_type( + tiledb_ctx_t* ctx, tiledb_current_domain_t* current_domain, tiledb_current_domain_type_t* type) TILEDB_NOEXCEPT; diff --git a/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc b/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc index f2f52fd19dfe..49780995b2c9 100644 --- a/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc +++ b/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc @@ -32,10 +32,10 @@ int main() { tiledb_current_domain_create(nullptr, nullptr); tiledb_current_domain_free(nullptr); - tiledb_current_domain_set_ndrectangle(nullptr, nullptr); - tiledb_current_domain_get_ndrectangle(nullptr, nullptr); - tiledb_current_domain_get_is_empty(nullptr, nullptr); - tiledb_current_domain_get_type(nullptr, nullptr); + tiledb_current_domain_set_ndrectangle(nullptr, nullptr, nullptr); + tiledb_current_domain_get_ndrectangle(nullptr, nullptr, nullptr); + tiledb_current_domain_get_is_empty(nullptr, nullptr, nullptr); + tiledb_current_domain_get_type(nullptr, nullptr, nullptr); return 0; } diff --git a/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc b/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc index d34db5fc5299..d9142af7ba80 100644 --- a/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc +++ b/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc @@ -100,17 +100,34 @@ TEST_CASE_METHOD( crd = nullptr; CHECK(tiledb_current_domain_create(ctx, &crd) == TILEDB_OK); - CHECK(tiledb_current_domain_set_ndrectangle(nullptr, nullptr) == TILEDB_ERR); - CHECK(tiledb_current_domain_set_ndrectangle(crd, nullptr) == TILEDB_ERR); + CHECK( + tiledb_current_domain_set_ndrectangle(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_current_domain_set_ndrectangle(ctx, nullptr, nullptr) == + TILEDB_ERR); + CHECK(tiledb_current_domain_set_ndrectangle(ctx, crd, nullptr) == TILEDB_ERR); - CHECK(tiledb_current_domain_get_ndrectangle(nullptr, nullptr) == TILEDB_ERR); - CHECK(tiledb_current_domain_get_ndrectangle(crd, nullptr) == TILEDB_ERR); + CHECK( + tiledb_current_domain_get_ndrectangle(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_current_domain_get_ndrectangle(ctx, nullptr, nullptr) == + TILEDB_ERR); + CHECK(tiledb_current_domain_get_ndrectangle(ctx, crd, nullptr) == TILEDB_ERR); - CHECK(tiledb_current_domain_get_is_empty(nullptr, nullptr) == TILEDB_ERR); - CHECK(tiledb_current_domain_get_is_empty(crd, nullptr) == TILEDB_ERR); + CHECK( + tiledb_current_domain_get_is_empty(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_current_domain_get_is_empty(ctx, nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_get_is_empty(ctx, crd, nullptr) == TILEDB_ERR); - CHECK(tiledb_current_domain_get_type(nullptr, nullptr) == TILEDB_ERR); - CHECK(tiledb_current_domain_get_type(crd, nullptr) == TILEDB_ERR); + CHECK( + tiledb_current_domain_get_type(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK(tiledb_current_domain_get_type(ctx, nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_get_type(ctx, crd, nullptr) == TILEDB_ERR); tiledb_current_domain_free(&crd); } @@ -126,22 +143,23 @@ TEST_CASE_METHOD( REQUIRE(tiledb_ndrectangle_alloc(ctx, domain_, &ndr) == TILEDB_OK); uint32_t is_empty = 0; - REQUIRE(tiledb_current_domain_get_is_empty(crd, &is_empty) == TILEDB_OK); + REQUIRE(tiledb_current_domain_get_is_empty(ctx, crd, &is_empty) == TILEDB_OK); CHECK(is_empty == 1); tiledb_current_domain_type_t type; - CHECK(tiledb_current_domain_get_type(crd, &type) == TILEDB_ERR); + CHECK(tiledb_current_domain_get_type(ctx, crd, &type) == TILEDB_ERR); - REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(ctx, crd, ndr) == TILEDB_OK); - REQUIRE(tiledb_current_domain_get_is_empty(crd, &is_empty) == TILEDB_OK); + REQUIRE(tiledb_current_domain_get_is_empty(ctx, crd, &is_empty) == TILEDB_OK); CHECK(is_empty == 0); - REQUIRE(tiledb_current_domain_get_type(crd, &type) == TILEDB_OK); + REQUIRE(tiledb_current_domain_get_type(ctx, crd, &type) == TILEDB_OK); CHECK(type == TILEDB_NDRECTANGLE); tiledb_ndrectangle_t* out_ndr = nullptr; - REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &out_ndr) == TILEDB_OK); + REQUIRE( + tiledb_current_domain_get_ndrectangle(ctx, crd, &out_ndr) == TILEDB_OK); CHECK(out_ndr != nullptr); // Verify that they point to the same tiledb::sm::NDRectangle instance. diff --git a/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc b/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc index 5acd46226e54..b4a4106dbbff 100644 --- a/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc +++ b/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc @@ -102,11 +102,7 @@ capi_return_t tiledb_ndrectangle_free(tiledb_ndrectangle_t** ndr) { } capi_return_t tiledb_ndrectangle_get_range_from_name( - tiledb_ctx_t* ctx, - tiledb_ndrectangle_t* ndr, - const char* name, - tiledb_range_t* range) { - ensure_context_is_valid(ctx); + tiledb_ndrectangle_t* ndr, const char* name, tiledb_range_t* range) { ensure_handle_is_valid(ndr); ensure_dim_name_is_valid(name); ensure_range_ptr_is_valid(range); @@ -119,11 +115,7 @@ capi_return_t tiledb_ndrectangle_get_range_from_name( } capi_return_t tiledb_ndrectangle_get_range( - tiledb_ctx_t* ctx, - tiledb_ndrectangle_t* ndr, - uint32_t idx, - tiledb_range_t* range) { - ensure_context_is_valid(ctx); + tiledb_ndrectangle_t* ndr, uint32_t idx, tiledb_range_t* range) { ensure_handle_is_valid(ndr); ensure_range_ptr_is_valid(range); @@ -134,11 +126,7 @@ capi_return_t tiledb_ndrectangle_get_range( } capi_return_t tiledb_ndrectangle_set_range_for_name( - tiledb_ctx_t* ctx, - tiledb_ndrectangle_t* ndr, - const char* name, - tiledb_range_t* range) { - ensure_context_is_valid(ctx); + tiledb_ndrectangle_t* ndr, const char* name, tiledb_range_t* range) { ensure_handle_is_valid(ndr); ensure_dim_name_is_valid(name); ensure_range_ptr_is_valid(range); @@ -157,11 +145,7 @@ capi_return_t tiledb_ndrectangle_set_range_for_name( } capi_return_t tiledb_ndrectangle_set_range( - tiledb_ctx_t* ctx, - tiledb_ndrectangle_t* ndr, - uint32_t idx, - tiledb_range_t* range) { - ensure_context_is_valid(ctx); + tiledb_ndrectangle_t* ndr, uint32_t idx, tiledb_range_t* range) { ensure_handle_is_valid(ndr); ensure_range_ptr_is_valid(range); @@ -177,8 +161,50 @@ capi_return_t tiledb_ndrectangle_set_range( return TILEDB_OK; } +capi_return_t tiledb_ndrectangle_get_dtype( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_datatype_t* type) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_output_pointer_is_valid(type); + + *type = static_cast(ndr->ndrectangle()->range_dtype(idx)); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_get_dtype_from_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_datatype_t* type) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_dim_name_is_valid(name); + ensure_output_pointer_is_valid(type); + + *type = static_cast( + ndr->ndrectangle()->range_dtype_for_name(name)); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_get_dim_num( + tiledb_ctx_t* ctx, tiledb_ndrectangle_t* ndr, uint32_t* ndim) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_output_pointer_is_valid(ndim); + + *ndim = ndr->ndrectangle()->domain()->dim_num(); + + return TILEDB_OK; +} + } // namespace tiledb::api +using tiledb::api::api_entry_context; using tiledb::api::api_entry_plain; using tiledb::api::api_entry_with_context; @@ -201,8 +227,7 @@ CAPI_INTERFACE( tiledb_ndrectangle_t* ndr, const char* name, tiledb_range_t* range) { - return api_entry_with_context< - tiledb::api::tiledb_ndrectangle_get_range_from_name>( + return api_entry_context( ctx, ndr, name, range); } @@ -212,7 +237,7 @@ CAPI_INTERFACE( tiledb_ndrectangle_t* ndr, uint32_t idx, tiledb_range_t* range) { - return api_entry_with_context( + return api_entry_context( ctx, ndr, idx, range); } @@ -222,8 +247,7 @@ CAPI_INTERFACE( tiledb_ndrectangle_t* ndr, const char* name, tiledb_range_t* range) { - return api_entry_with_context< - tiledb::api::tiledb_ndrectangle_set_range_for_name>( + return api_entry_context( ctx, ndr, name, range); } @@ -233,6 +257,36 @@ CAPI_INTERFACE( tiledb_ndrectangle_t* ndr, uint32_t idx, tiledb_range_t* range) { - return api_entry_with_context( + return api_entry_context( ctx, ndr, idx, range); } + +CAPI_INTERFACE( + ndrectangle_get_dtype, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_datatype_t* type) { + return api_entry_with_context( + ctx, ndr, idx, type); +} + +CAPI_INTERFACE( + ndrectangle_get_dtype_from_name, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_datatype_t* type) { + return api_entry_with_context< + tiledb::api::tiledb_ndrectangle_get_dtype_from_name>( + ctx, ndr, name, type); +} + +CAPI_INTERFACE( + ndrectangle_get_dim_num, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t* ndim) { + return api_entry_with_context( + ctx, ndr, ndim); +} diff --git a/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h b/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h index e0f80ea22b87..831494179a07 100644 --- a/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h +++ b/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h @@ -86,6 +86,7 @@ TILEDB_EXPORT capi_return_t tiledb_ndrectangle_alloc( * tiledb_ndrectangle_free(&ndr); * @endcode * + * @param ctx The TileDB context * @param ndr The n-dimensional rectangle to be freed * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ @@ -198,6 +199,73 @@ TILEDB_EXPORT capi_return_t tiledb_ndrectangle_set_range( uint32_t idx, tiledb_range_t* range) TILEDB_NOEXCEPT; +/** + * Get the TileDB datatype for dimension at idx from + * the N-dimensional rectangle passed as argument + * + * **Example:** + * + * @code{.c} + * tiledb_datatype_t type; + * tiledb_ndrectangle_get_dtype(ctx, ndr, 1, &type); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param idx The index of the dimension + * @param type The datatype to be returned + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_get_dtype( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_datatype_t* type) TILEDB_NOEXCEPT; + +/** + * Get the TileDB datatype for dimension name from + * the N-dimensional rectangle passed as argument + * + * **Example:** + * + * @code{.c} + * tiledb_datatype_t type; + * tiledb_ndrectangle_get_dtype_from_name(ctx, ndr, "dim1", &type); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param name The dimension name + * @param type The datatype to be returned + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_get_dtype_from_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_datatype_t* type) TILEDB_NOEXCEPT; + +/** + * Get the the number of dimensions of + * the N-dimensional rectangle passed as argument + * + * **Example:** + * + * @code{.c} + * uint32_t ndim; + * tiledb_ndrectangle_get_dim_num(ctx, ndr, &ndim); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param ndim The number of dimensions to be returned + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_get_dim_num( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t* ndim) TILEDB_NOEXCEPT; + #ifdef __cplusplus } #endif diff --git a/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc b/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc index 49221b0b3c99..4d5d10ee7a32 100644 --- a/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc +++ b/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc @@ -144,6 +144,33 @@ TEST_CASE_METHOD( tiledb_ndrectangle_set_range_for_name(ctx, ndr, "doesntexist", &range) == TILEDB_ERR); + CHECK( + tiledb_ndrectangle_get_dtype(nullptr, nullptr, 0, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK(tiledb_ndrectangle_get_dtype(ctx, nullptr, 0, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_get_dtype(ctx, ndr, 0, nullptr) == TILEDB_ERR); + tiledb_datatype_t dtype; + CHECK(tiledb_ndrectangle_get_dtype(ctx, ndr, 2, &dtype) == TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_get_dtype_from_name( + nullptr, nullptr, "dim1", nullptr) == TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_ndrectangle_get_dtype_from_name(ctx, nullptr, "dim1", nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_get_dtype_from_name(ctx, ndr, "dim1", nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_get_dtype_from_name(ctx, ndr, "doesntexist", &dtype) == + TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_get_dim_num(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK(tiledb_ndrectangle_get_dim_num(ctx, nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_get_dim_num(ctx, ndr, nullptr) == TILEDB_ERR); + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); } @@ -186,5 +213,17 @@ TEST_CASE_METHOD( CHECK(std::memcmp(range.min, out_range_d2.min, range.min_size) == 0); CHECK(std::memcmp(range.max, out_range_d2.max, range.max_size) == 0); + tiledb_datatype_t dtype; + REQUIRE(tiledb_ndrectangle_get_dtype(ctx, ndr, 0, &dtype) == TILEDB_OK); + CHECK(dtype == TILEDB_UINT64); + REQUIRE( + tiledb_ndrectangle_get_dtype_from_name(ctx, ndr, "d1", &dtype) == + TILEDB_OK); + CHECK(dtype == TILEDB_UINT64); + + uint32_t ndim; + REQUIRE(tiledb_ndrectangle_get_dim_num(ctx, ndr, &ndim) == TILEDB_OK); + CHECK(ndim == 2); + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); } diff --git a/tiledb/api/c_api_test_support/testsupport_capi_array_schema.h b/tiledb/api/c_api_test_support/testsupport_capi_array_schema.h new file mode 100644 index 000000000000..420bb8621b7b --- /dev/null +++ b/tiledb/api/c_api_test_support/testsupport_capi_array_schema.h @@ -0,0 +1,93 @@ +/** + * @file tiledb/api/c_api_test_support/testsupport_capi_array_schema.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines test support for the array schema section of the C API. + */ + +#ifndef TILEDB_TESTSUPPORT_CAPI_ARRAY_SCHEMA_H +#define TILEDB_TESTSUPPORT_CAPI_ARRAY_SCHEMA_H + +#include "testsupport_capi_context.h" +#include "tiledb/api/c_api/array_schema/array_schema_api_external.h" +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" + +namespace tiledb::api::test_support { + +/** + * Ordinary array schema base class + */ +class ordinary_array_schema { + public: + ordinary_context context{}; + tiledb_array_schema_handle_t* schema{nullptr}; + + ordinary_array_schema(tiledb_array_type_t array_type = TILEDB_SPARSE) { + auto rc = tiledb_array_schema_alloc(context.context, array_type, &schema); + if (rc != TILEDB_OK) { + throw std::runtime_error("error creating test array_schema"); + } + if (schema == nullptr) { + throw std::logic_error( + "tiledb_array_schema_alloc returned OK but without array_schema"); + } + } + + ~ordinary_array_schema() { + tiledb_array_schema_free(&schema); + } + + [[nodiscard]] tiledb_ctx_handle_t* ctx() const { + return context.context; + } +}; + +struct ordinary_array_schema_with_attr : public ordinary_array_schema { + tiledb_attribute_t* attr{nullptr}; + + ordinary_array_schema_with_attr( + tiledb_array_type_t array_type = TILEDB_SPARSE) + : ordinary_array_schema(array_type) { + auto rc = tiledb_attribute_alloc(context.context, "a", TILEDB_INT32, &attr); + if (rc != TILEDB_OK) { + throw std::runtime_error("error creating test attribute"); + } + rc = tiledb_array_schema_add_attribute(context.context, schema, attr); + if (rc != TILEDB_OK) { + throw std::runtime_error("error adding test attribute to test schema"); + } + } + + ~ordinary_array_schema_with_attr() { + tiledb_attribute_free(&attr); + } +}; + +} // namespace tiledb::api::test_support + +#endif // TILEDB_TESTSUPPORT_CAPI_ARRAY_SCHEMA_H diff --git a/tiledb/common/common-std.h b/tiledb/common/common-std.h index dc7315818db1..fe91c90b5f6d 100644 --- a/tiledb/common/common-std.h +++ b/tiledb/common/common-std.h @@ -83,11 +83,8 @@ using std::tuple; using std::nullopt; using std::optional; -/* - * Span implementation, similar to std::span of C++20 - */ -#include "external/include/span/span.hpp" -using tcb::span; +#include +using std::span; using std::size_t; diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index 8768bc04b2c5..b974dede4a30 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -76,9 +76,9 @@ author = 'TileDB, Inc.' # The short X.Y version. -version = '2.26' +version = '2.27' # The full version, including alpha/beta/rc tags. -release = '2.26.0' +release = '2.27.0' # Breathe extension configuration. doxygen_xml_dir = os.path.join(TILEDB_BUILD_DIR, 'xml/') diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index c37682971d39..231440197334 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -804,11 +804,17 @@ void Array::encryption_type( shared_ptr Array::get_enumeration( const std::string& enumeration_name) { - return get_enumerations({enumeration_name})[0]; + if (!is_open_) { + throw ArrayException("Unable to load enumerations; Array is not open."); + } + + return get_enumerations( + {enumeration_name}, opened_array_->array_schema_latest_ptr())[0]; } std::vector> Array::get_enumerations( - const std::vector& enumeration_names) { + const std::vector& enumeration_names, + shared_ptr schema) { if (!is_open_) { throw ArrayException("Unable to load enumerations; Array is not open."); } @@ -816,7 +822,7 @@ std::vector> Array::get_enumerations( // Dedupe requested names and filter out anything already loaded. std::unordered_set enmrs_to_load; for (auto& enmr_name : enumeration_names) { - if (array_schema_latest().is_enumeration_loaded(enmr_name)) { + if (schema->is_enumeration_loaded(enmr_name)) { continue; } enmrs_to_load.insert(enmr_name); @@ -850,7 +856,7 @@ std::vector> Array::get_enumerations( // Create a vector of paths to be loaded. std::vector paths_to_load; for (auto& enmr_name : enmrs_to_load) { - auto path = array_schema_latest().get_enumeration_path_name(enmr_name); + auto path = schema->get_enumeration_path_name(enmr_name); paths_to_load.push_back(path); } @@ -861,14 +867,14 @@ std::vector> Array::get_enumerations( // Store the loaded enumerations in the schema for (auto& enmr : loaded) { - opened_array_->array_schema_latest_ptr()->store_enumeration(enmr); + schema->store_enumeration(enmr); } } // Return the requested list of enumerations std::vector> ret(enumeration_names.size()); for (size_t i = 0; i < enumeration_names.size(); i++) { - ret[i] = array_schema_latest().get_enumeration(enumeration_names[i]); + ret[i] = schema->get_enumeration(enumeration_names[i]); } return ret; } @@ -878,7 +884,9 @@ void Array::load_all_enumerations() { throw ArrayException("Unable to load all enumerations; Array is not open."); } // Load all enumerations, discarding the returned list of loaded enumerations. - get_enumerations(array_schema_latest().get_enumeration_names()); + for (const auto& schema : array_schemas_all()) { + get_enumerations(schema.second->get_enumeration_names(), schema.second); + } } bool Array::is_empty() const { diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index bf6b3292daae..c2ece7e384f5 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -589,10 +589,12 @@ class Array { * loaded before this function returns. * * @param enumeration_names The names of the enumerations. + * @param schema The ArraySchema to store loaded enumerations in. * @return std::vector> The loaded enumerations. */ std::vector> get_enumerations( - const std::vector& enumeration_names); + const std::vector& enumeration_names, + shared_ptr schema); /** Load all enumerations for the array. */ void load_all_enumerations(); diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 32f966b46063..444036d9f2ee 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -123,7 +123,7 @@ shared_ptr ArrayDirectory::load_array_schema_latest( auto&& array_schema = load_array_schema_from_uri( resources_.get(), schema_uri, encryption_key, memory_tracker); - array_schema->set_array_uri(uri_); + array_schema->set_array_uri(uri_.remove_trailing_slash()); return std::move(array_schema); } diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index 831d51d21f4d..9a398c660c33 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -98,7 +98,7 @@ conclude(object_library) # commence(object_library array_schema_operations) this_target_sources(array_schema_operations.cc) - this_target_object_libraries(array_schema generic_tile_io tiledb_crypto) + this_target_object_libraries(array_directory array_schema generic_tile_io tiledb_crypto) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 24df5692c67e..1de4e2982f8b 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -1150,7 +1150,7 @@ void ArraySchema::drop_enumeration(const std::string& enmr_name) { } if (attr_enmr_name.value() == enmr_name) { throw ArraySchemaException( - "Unable to drop enumeration '" + enmr_name + "' as it is used by " + + "Unable to drop enumeration '" + enmr_name + "' as it is used by" + " attribute '" + attr->name() + "'."); } } diff --git a/tiledb/sm/array_schema/array_schema_operations.cc b/tiledb/sm/array_schema/array_schema_operations.cc index 6fdf2384af9f..d17388b48d62 100644 --- a/tiledb/sm/array_schema/array_schema_operations.cc +++ b/tiledb/sm/array_schema/array_schema_operations.cc @@ -36,8 +36,12 @@ #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/config/config.h" +#include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/integral_type_casts.h" +#include "tiledb/sm/rest/rest_client.h" +#include "tiledb/sm/storage_manager/context.h" #include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" @@ -227,4 +231,61 @@ void store_array_schema( } } +shared_ptr load_array_schema( + const Context& ctx, const URI& uri, const Config& config) { + // Check array name + if (uri.is_invalid()) { + throw std::runtime_error("Failed to load array schema; Invalid array URI"); + } + + if (uri.is_tiledb()) { + auto& rest_client = ctx.rest_client(); + auto&& [st, array_schema_response] = + rest_client.get_array_schema_from_rest(uri); + throw_if_not_ok(st); + return std::move(array_schema_response).value(); + } else { + // Create key + tiledb::sm::EncryptionKey key; + throw_if_not_ok( + key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); + + // Load URIs from the array directory + optional array_dir; + array_dir.emplace( + ctx.resources(), + uri, + 0, + UINT64_MAX, + tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY); + + auto tracker = ctx.resources().ephemeral_memory_tracker(); + // Load latest array schema + auto&& array_schema_latest = + array_dir->load_array_schema_latest(key, tracker); + + // Load enumerations if config option is set. + bool incl_enums = config.get( + "rest.load_enumerations_on_array_open", Config::must_find); + if (incl_enums) { + std::vector enmr_paths_to_load; + auto enmr_names = array_schema_latest->get_enumeration_names(); + for (auto& name : enmr_names) { + if (!array_schema_latest->is_enumeration_loaded(name)) { + auto& path = array_schema_latest->get_enumeration_path_name(name); + enmr_paths_to_load.emplace_back(path); + } + } + + auto enmrs_loaded = array_dir->load_enumerations_from_paths( + enmr_paths_to_load, key, tracker); + for (auto& enmr : enmrs_loaded) { + array_schema_latest->store_enumeration(enmr); + } + } + + return std::move(array_schema_latest); + } +} + } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/array_schema_operations.h b/tiledb/sm/array_schema/array_schema_operations.h index 19a230bc366e..64c1adbd0b5e 100644 --- a/tiledb/sm/array_schema/array_schema_operations.h +++ b/tiledb/sm/array_schema/array_schema_operations.h @@ -41,8 +41,11 @@ using namespace tiledb::common; namespace tiledb::sm { class ArraySchema; +class Config; +class Context; class ContextResources; class EncryptionKey; +class URI; /* ********************************* */ /* API */ @@ -76,6 +79,16 @@ void store_array_schema( const shared_ptr& array_schema, const EncryptionKey& encryption_key); +/** + * Handles loading an array schema from a URI. + * + * @param ctx TileDB Context. + * @param uri The URI of the array to load schema. + * @param config TileDB Config. If null, the context config will be used. + */ +shared_ptr load_array_schema( + const Context& ctx, const URI& uri, const Config& config); + } // namespace tiledb::sm #endif // TILEDB_ARRAY_SCHEMA_OPERATIONS_H diff --git a/tiledb/sm/array_schema/ndrectangle.cc b/tiledb/sm/array_schema/ndrectangle.cc index 113ab2ac747c..c25ae9ee43f8 100644 --- a/tiledb/sm/array_schema/ndrectangle.cc +++ b/tiledb/sm/array_schema/ndrectangle.cc @@ -167,6 +167,20 @@ const Range& NDRectangle::get_range_for_name(const std::string& name) const { return get_range(idx); } +Datatype NDRectangle::range_dtype(uint32_t idx) const { + if (idx >= range_data_.size()) { + throw std::logic_error( + "The index does not correspond to a valid dimension in the " + "NDRectangle"); + } + return domain()->dimension_ptr(idx)->type(); +} + +Datatype NDRectangle::range_dtype_for_name(const std::string& name) const { + auto idx = domain()->get_dimension_index(name); + return range_dtype(idx); +} + } // namespace tiledb::sm std::ostream& operator<<(std::ostream& os, const tiledb::sm::NDRectangle& ndr) { diff --git a/tiledb/sm/array_schema/ndrectangle.h b/tiledb/sm/array_schema/ndrectangle.h index c7e363fd47c9..9eee56f32e01 100644 --- a/tiledb/sm/array_schema/ndrectangle.h +++ b/tiledb/sm/array_schema/ndrectangle.h @@ -169,6 +169,7 @@ class NDRectangle { /** * Get the range for the dimension at idx * + * @param name The dimension index * @return The range of the dimension */ const Range& get_range(uint32_t idx) const; @@ -176,10 +177,27 @@ class NDRectangle { /** * Get the range for the dimension specified by name * + * @param name The name of the dimension * @return The range of the dimension */ const Range& get_range_for_name(const std::string& name) const; + /** + * Get the data type of the range at idx + * + * @param idx The index of the range + * @return The range datatype + */ + Datatype range_dtype(uint32_t idx) const; + + /** + * Get the data type of the range at idx + * + * @param name The dimension name + * @return The range datatype + */ + Datatype range_dtype_for_name(const std::string& name) const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/c_api/api_argument_validator.h b/tiledb/sm/c_api/api_argument_validator.h index df62b27c37b8..1f15be6b4c6f 100644 --- a/tiledb/sm/c_api/api_argument_validator.h +++ b/tiledb/sm/c_api/api_argument_validator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -81,17 +81,6 @@ inline void ensure_subarray_is_valid(const tiledb_subarray_t* p) { } } // namespace tiledb::api -inline int32_t sanity_check( - tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema) { - if (array_schema == nullptr || array_schema->array_schema_ == nullptr) { - auto st = Status_Error("Invalid TileDB array schema object"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } - return TILEDB_OK; -} - inline int32_t sanity_check( tiledb_ctx_t* ctx, const tiledb_array_schema_evolution_t* schema_evolution) { diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index f45043b5ca27..bd96d9952065 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -36,6 +36,8 @@ #include "tiledb_serialization.h" #include "tiledb_struct_def.h" +#include "tiledb/api/c_api/array/array_api_internal.h" +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" #include "tiledb/api/c_api/attribute/attribute_api_internal.h" #include "tiledb/api/c_api/buffer/buffer_api_internal.h" #include "tiledb/api/c_api/buffer_list/buffer_list_api_internal.h" @@ -55,17 +57,14 @@ #include "tiledb/common/logger.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" -#include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/c_api/api_argument_validator.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/config/config_iter.h" #include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/cpp_api/core_interface.h" -#include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/enums/filesystem.h" -#include "tiledb/sm/enums/layout.h" #include "tiledb/sm/enums/mime_type.h" #include "tiledb/sm/enums/object_type.h" #include "tiledb/sm/enums/query_status.h" @@ -136,37 +135,6 @@ namespace tiledb::api { /* ENUMS TO/FROM STR */ /* ****************************** */ -int32_t tiledb_array_type_to_str( - tiledb_array_type_t array_type, const char** str) { - const auto& strval = - tiledb::sm::array_type_str((tiledb::sm::ArrayType)array_type); - *str = strval.c_str(); - return strval.empty() ? TILEDB_ERR : TILEDB_OK; -} - -int32_t tiledb_array_type_from_str( - const char* str, tiledb_array_type_t* array_type) { - tiledb::sm::ArrayType val = tiledb::sm::ArrayType::DENSE; - if (!tiledb::sm::array_type_enum(str, &val).ok()) - return TILEDB_ERR; - *array_type = (tiledb_array_type_t)val; - return TILEDB_OK; -} - -int32_t tiledb_layout_to_str(tiledb_layout_t layout, const char** str) { - const auto& strval = tiledb::sm::layout_str((tiledb::sm::Layout)layout); - *str = strval.c_str(); - return strval.empty() ? TILEDB_ERR : TILEDB_OK; -} - -int32_t tiledb_layout_from_str(const char* str, tiledb_layout_t* layout) { - tiledb::sm::Layout val = tiledb::sm::Layout::ROW_MAJOR; - if (!tiledb::sm::layout_enum(str, &val).ok()) - return TILEDB_ERR; - *layout = (tiledb_layout_t)val; - return TILEDB_OK; -} - int32_t tiledb_encryption_type_to_str( tiledb_encryption_type_t encryption_type, const char** str) { const auto& strval = tiledb::sm::encryption_type_str( @@ -221,548 +189,25 @@ int32_t tiledb_serialization_type_from_str( /* ********************************* */ /* LOGGING */ -/* ********************************* */ - -capi_return_t tiledb_log_warn(tiledb_ctx_t* ctx, const char* message) { - if (message == nullptr) { - return TILEDB_ERR; - } - - auto logger = ctx->resources().logger(); - logger->warn(message); - - return TILEDB_OK; -} - -/* ********************************* */ -/* AS BUILT */ -/* ********************************* */ -capi_return_t tiledb_as_built_dump(tiledb_string_t** out) { - ensure_output_pointer_is_valid(out); - *out = tiledb_string_handle_t::make_handle(as_built::dump()); - return TILEDB_OK; -} - -/* ****************************** */ -/* ARRAY SCHEMA */ -/* ****************************** */ - -int32_t tiledb_array_schema_alloc( - tiledb_ctx_t* ctx, - tiledb_array_type_t array_type, - tiledb_array_schema_t** array_schema) { - // Create array schema struct - *array_schema = new (std::nothrow) tiledb_array_schema_t; - if (*array_schema == nullptr) { - auto st = Status_Error("Failed to allocate TileDB array schema object"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_OOM; - } - - // Create a new ArraySchema object - auto memory_tracker = ctx->resources().create_memory_tracker(); - memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_CREATE); - (*array_schema)->array_schema_ = make_shared( - HERE(), static_cast(array_type), memory_tracker); - if ((*array_schema)->array_schema_ == nullptr) { - auto st = Status_Error("Failed to allocate TileDB array schema object"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_OOM; - } - - // Success - return TILEDB_OK; -} - -void tiledb_array_schema_free(tiledb_array_schema_t** array_schema) { - if (array_schema != nullptr && *array_schema != nullptr) { - delete *array_schema; - *array_schema = nullptr; - } -} - -int32_t tiledb_array_schema_add_attribute( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_attribute_t* attr) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - ensure_attribute_is_valid(attr); - /** Note: The call to make_shared creates a copy of the attribute and - * the user-visible handle to the attr no longer refers to the same object - * that's in the array_schema. - **/ - throw_if_not_ok( - array_schema->array_schema_->add_attribute(attr->copy_attribute())); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_set_allows_dups( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, int allows_dups) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - throw_if_not_ok(array_schema->array_schema_->set_allows_dups(allows_dups)); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_allows_dups( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, int* allows_dups) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - *allows_dups = (int)array_schema->array_schema_->allows_dups(); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_version( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, uint32_t* version) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - *version = (uint32_t)array_schema->array_schema_->version(); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_set_domain( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_domain_t* domain) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - throw_if_not_ok( - array_schema->array_schema_->set_domain(domain->copy_domain())); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_set_capacity( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, uint64_t capacity) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - array_schema->array_schema_->set_capacity(capacity); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_set_cell_order( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_layout_t cell_order) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - throw_if_not_ok(array_schema->array_schema_->set_cell_order( - static_cast(cell_order))); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_set_tile_order( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_layout_t tile_order) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - throw_if_not_ok(array_schema->array_schema_->set_tile_order( - static_cast(tile_order))); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_timestamp_range( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - uint64_t* lo, - uint64_t* hi) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - - auto timestamp_range = array_schema->array_schema_->timestamp_range(); - *lo = std::get<0>(timestamp_range); - *hi = std::get<1>(timestamp_range); - - return TILEDB_OK; -} - -int32_t tiledb_array_schema_add_enumeration( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_enumeration_t* enumeration) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - - api::ensure_enumeration_is_valid(enumeration); - - array_schema->array_schema_->add_enumeration(enumeration->copy()); - - return TILEDB_OK; -} - -int32_t tiledb_array_schema_set_coords_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - api::ensure_filter_list_is_valid(filter_list); - - throw_if_not_ok(array_schema->array_schema_->set_coords_filter_pipeline( - filter_list->pipeline())); - - return TILEDB_OK; -} - -int32_t tiledb_array_schema_set_offsets_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - api::ensure_filter_list_is_valid(filter_list); - - throw_if_not_ok( - array_schema->array_schema_->set_cell_var_offsets_filter_pipeline( - filter_list->pipeline())); - - return TILEDB_OK; -} - -int32_t tiledb_array_schema_set_validity_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - api::ensure_filter_list_is_valid(filter_list); - - throw_if_not_ok( - array_schema->array_schema_->set_cell_validity_filter_pipeline( - filter_list->pipeline())); - - return TILEDB_OK; -} - -int32_t tiledb_array_schema_check( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - array_schema->array_schema_->check(ctx->resources().config()); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_load( - tiledb_ctx_t* ctx, - const char* array_uri, - tiledb_array_schema_t** array_schema) { - // Create array schema - *array_schema = new (std::nothrow) tiledb_array_schema_t; - if (*array_schema == nullptr) { - auto st = Status_Error("Failed to allocate TileDB array schema object"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_OOM; - } - - // Check array name - tiledb::sm::URI uri(array_uri); - if (uri.is_invalid()) { - auto st = Status_Error("Failed to load array schema; Invalid array URI"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } - - if (uri.is_tiledb()) { - auto& rest_client = ctx->context().rest_client(); - auto&& [st, array_schema_rest] = - rest_client.get_array_schema_from_rest(uri); - if (!st.ok()) { - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - delete *array_schema; - return TILEDB_ERR; - } - (*array_schema)->array_schema_ = array_schema_rest.value(); - } else { - // Create key - tiledb::sm::EncryptionKey key; - throw_if_not_ok( - key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); - - // Load URIs from the array directory - optional array_dir; - try { - array_dir.emplace( - ctx->resources(), - uri, - 0, - UINT64_MAX, - tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY); - } catch (const std::logic_error& le) { - auto st = Status_ArrayDirectoryError(le.what()); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - delete *array_schema; - return TILEDB_ERR; - } - - auto tracker = ctx->resources().ephemeral_memory_tracker(); - - // Load latest array schema - auto&& array_schema_latest = - array_dir->load_array_schema_latest(key, tracker); - (*array_schema)->array_schema_ = array_schema_latest; - } - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_array_type( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_array_type_t* array_type) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - *array_type = static_cast( - array_schema->array_schema_->array_type()); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_capacity( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - uint64_t* capacity) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - *capacity = array_schema->array_schema_->capacity(); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_cell_order( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_layout_t* cell_order) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - *cell_order = - static_cast(array_schema->array_schema_->cell_order()); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_coords_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - api::ensure_output_pointer_is_valid(filter_list); - // Copy-construct a separate FilterPipeline object - *filter_list = tiledb_filter_list_t::make_handle( - sm::FilterPipeline{array_schema->array_schema_->coords_filters()}); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_offsets_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - api::ensure_output_pointer_is_valid(filter_list); - // Copy-construct a separate FilterPipeline object - *filter_list = tiledb_filter_list_t::make_handle(sm::FilterPipeline{ - array_schema->array_schema_->cell_var_offsets_filters()}); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_validity_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - api::ensure_output_pointer_is_valid(filter_list); - // Copy-construct a separate FilterPipeline object - *filter_list = tiledb_filter_list_t::make_handle( - sm::FilterPipeline{array_schema->array_schema_->cell_validity_filters()}); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_domain( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_domain_t** domain) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - ensure_output_pointer_is_valid(domain); - *domain = tiledb_domain_handle_t::make_handle( - array_schema->array_schema_->shared_domain()); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_tile_order( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_layout_t* tile_order) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - *tile_order = - static_cast(array_schema->array_schema_->tile_order()); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_attribute_num( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - uint32_t* attribute_num) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - *attribute_num = array_schema->array_schema_->attribute_num(); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_dump( - tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, FILE* out) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; - - ensure_cstream_handle_is_valid(out); - - std::stringstream ss; - - ss << *array_schema->array_schema_; - size_t r = fwrite(ss.str().c_str(), sizeof(char), ss.str().size(), out); - if (r != ss.str().size()) { - throw CAPIException( - "Error writing array schema " + - array_schema->array_schema_->array_uri().to_string() + " to file"); - } - - return TILEDB_OK; -} - -int32_t tiledb_array_schema_dump_str( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_string_t** out) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - ensure_output_pointer_is_valid(out); - std::stringstream ss; - ss << *array_schema->array_schema_; - *out = tiledb_string_handle_t::make_handle(ss.str()); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_attribute_from_index( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - uint32_t index, - tiledb_attribute_t** attr) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - ensure_output_pointer_is_valid(attr); - uint32_t attribute_num = array_schema->array_schema_->attribute_num(); - if (attribute_num == 0) { - *attr = nullptr; - return TILEDB_OK; - } - if (index >= attribute_num) { - std::ostringstream errmsg; - errmsg << "Attribute index: " << index << " out of bounds given " - << attribute_num << " attributes in array " - << array_schema->array_schema_->array_uri().to_string(); - auto st = Status_ArraySchemaError(errmsg.str()); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } - - auto found_attr = array_schema->array_schema_->shared_attribute(index); - if (!found_attr) { - throw CAPIStatusException("Attribute not found, but index is valid!"); - } - *attr = tiledb_attribute_handle_t::make_handle(found_attr); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_get_attribute_from_name( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - const char* name, - tiledb_attribute_t** attr) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - ensure_output_pointer_is_valid(attr); - uint32_t attribute_num = array_schema->array_schema_->attribute_num(); - if (attribute_num == 0) { - *attr = nullptr; - return TILEDB_OK; - } - std::string name_string(name); - auto found_attr = array_schema->array_schema_->shared_attribute(name_string); - if (!found_attr) { - throw CAPIException( - std::string("Attribute name: ") + - (name_string.empty() ? "" : name) + - " does not exist for array " + - array_schema->array_schema_->array_uri().to_string()); - } - *attr = tiledb_attribute_handle_t::make_handle(found_attr); - return TILEDB_OK; -} - -int32_t tiledb_array_schema_has_attribute( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - const char* name, - int32_t* has_attr) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - - bool b; - throw_if_not_ok(array_schema->array_schema_->has_attribute(name, &b)); - - *has_attr = b ? 1 : 0; - - return TILEDB_OK; -} - -int32_t tiledb_array_schema_set_current_domain( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_current_domain_t* current_domain) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { - return TILEDB_ERR; - } - - api::ensure_handle_is_valid(current_domain); - - array_schema->array_schema_->set_current_domain( - current_domain->current_domain()); - - return TILEDB_OK; -} +/* ********************************* */ -int32_t tiledb_array_schema_get_current_domain( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_current_domain_t** current_domain) { - if (sanity_check(ctx, array_schema) == TILEDB_ERR) { +capi_return_t tiledb_log_warn(tiledb_ctx_t* ctx, const char* message) { + if (message == nullptr) { return TILEDB_ERR; } - api::ensure_output_pointer_is_valid(current_domain); + auto logger = ctx->resources().logger(); + logger->warn(message); - // There is always a current domain on an ArraySchema instance, - // when none was set explicitly, there is an empty current domain. - *current_domain = tiledb_current_domain_handle_t::make_handle( - array_schema->array_schema_->get_current_domain()); + return TILEDB_OK; +} +/* ********************************* */ +/* AS BUILT */ +/* ********************************* */ +capi_return_t tiledb_as_built_dump(tiledb_string_t** out) { + ensure_output_pointer_is_valid(out); + *out = tiledb_string_handle_t::make_handle(as_built::dump()); return TILEDB_OK; } @@ -1992,35 +1437,16 @@ int32_t tiledb_array_get_open_timestamp_end( int32_t tiledb_array_delete(tiledb_ctx_t* ctx, const char* uri) { // Allocate an array object - tiledb_array_t* array = new (std::nothrow) tiledb_array_t; - try { - array->array_ = make_shared( - HERE(), ctx->resources(), tiledb::sm::URI(uri)); - } catch (std::bad_alloc&) { - auto st = Status_Error( - "Failed to create TileDB array object; Memory allocation error"); - delete array; - array = nullptr; - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_OOM; - } - + auto array = make_shared( + HERE(), ctx->resources(), tiledb::sm::URI(uri)); // Open the array for exclusive modification - throw_if_not_ok(array->array_->open( + throw_if_not_ok(array->open( tiledb::sm::QueryType::MODIFY_EXCLUSIVE, tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); - try { - array->array_->delete_array(tiledb::sm::URI(uri)); - } catch (std::exception& e) { - auto st = Status_ArrayError(e.what()); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } + array->delete_array(tiledb::sm::URI(uri)); return TILEDB_OK; } @@ -2032,27 +1458,18 @@ capi_return_t tiledb_array_delete_fragments_v2( uint64_t timestamp_end) { auto uri = tiledb::sm::URI(uri_str); if (uri.is_invalid()) { - throw api::CAPIStatusException( - "Failed to delete fragments; Invalid input uri"); + throw CAPIException("Failed to delete fragments; Invalid input uri"); } // Allocate an array object - tiledb_array_t* array = new (std::nothrow) tiledb_array_t; - try { - array->array_ = - make_shared(HERE(), ctx->resources(), uri); - } catch (...) { - delete array; - array = nullptr; - throw api::CAPIStatusException("Failed to create array"); - } + auto array = make_shared(HERE(), ctx->resources(), uri); // Set array open timestamps - array->array_->set_timestamp_start(timestamp_start); - array->array_->set_timestamp_end(timestamp_end); + array->set_timestamp_start(timestamp_start); + array->set_timestamp_end(timestamp_end); // Open the array for exclusive modification - throw_if_not_ok(array->array_->open( + throw_if_not_ok(array->open( static_cast(TILEDB_MODIFY_EXCLUSIVE), static_cast(TILEDB_NO_ENCRYPTION), nullptr, @@ -2060,18 +1477,14 @@ capi_return_t tiledb_array_delete_fragments_v2( // Delete fragments try { - array->array_->delete_fragments(uri, timestamp_start, timestamp_end); + array->delete_fragments(uri, timestamp_start, timestamp_end); } catch (...) { - throw_if_not_ok(array->array_->close()); - delete array; - array = nullptr; - throw api::CAPIStatusException("Failed to delete fragments"); + throw_if_not_ok(array->close()); + throw CAPIException("Failed to delete fragments"); } // Close and delete the array - throw_if_not_ok(array->array_->close()); - delete array; - array = nullptr; + throw_if_not_ok(array->close()); return TILEDB_OK; } @@ -2083,18 +1496,17 @@ capi_return_t tiledb_array_delete_fragments_list( const size_t num_fragments) { auto uri = tiledb::sm::URI(uri_str); if (uri.is_invalid()) { - throw api::CAPIStatusException( - "Failed to delete_fragments_list; Invalid input uri"); + throw CAPIException("Failed to delete_fragments_list; Invalid input uri"); } if (num_fragments < 1) { - throw api::CAPIStatusException( + throw CAPIException( "Failed to delete_fragments_list; Invalid input number of fragments"); } for (size_t i = 0; i < num_fragments; i++) { if (tiledb::sm::URI(fragment_uris[i]).is_invalid()) { - throw api::CAPIStatusException( + throw CAPIException( "Failed to delete_fragments_list; Invalid input fragment uri"); } } @@ -2107,37 +1519,25 @@ capi_return_t tiledb_array_delete_fragments_list( } // Allocate an array object - tiledb_array_t* array = new (std::nothrow) tiledb_array_t; - try { - array->array_ = - make_shared(HERE(), ctx->resources(), uri); - } catch (...) { - delete array; - array = nullptr; - throw api::CAPIStatusException("Failed to create array"); - } + auto array = make_shared(HERE(), ctx->resources(), uri); // Open the array for exclusive modification - throw_if_not_ok(array->array_->open( - static_cast(TILEDB_MODIFY_EXCLUSIVE), - static_cast(TILEDB_NO_ENCRYPTION), + throw_if_not_ok(array->open( + tiledb::sm::QueryType::MODIFY_EXCLUSIVE, + tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Delete fragments list try { - array->array_->delete_fragments_list(uris); + array->delete_fragments_list(uris); } catch (...) { - throw_if_not_ok(array->array_->close()); - delete array; - array = nullptr; - throw api::CAPIStatusException("Failed to delete fragments_list"); + throw_if_not_ok(array->close()); + throw CAPIException("Failed to delete fragments_list"); } // Close the array - throw_if_not_ok(array->array_->close()); - delete array; - array = nullptr; + throw_if_not_ok(array->close()); return TILEDB_OK; } @@ -2220,25 +1620,16 @@ int32_t tiledb_array_get_schema( if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; } - - *array_schema = new (std::nothrow) tiledb_array_schema_t; - if (*array_schema == nullptr) { - auto st = Status_Error("Failed to allocate TileDB array schema"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_OOM; - } + ensure_output_pointer_is_valid(array_schema); // Get schema auto&& [st, array_schema_get] = array->array_->get_array_schema(); if (!st.ok()) { LOG_STATUS_NO_RETURN_VALUE(st); save_error(ctx, st); - delete *array_schema; - *array_schema = nullptr; return TILEDB_ERR; } - (*array_schema)->array_schema_ = array_schema_get.value(); + *array_schema = tiledb_array_schema_t::make_handle(array_schema_get.value()); return TILEDB_OK; } @@ -2267,9 +1658,7 @@ int32_t tiledb_array_create( tiledb_ctx_t* ctx, const char* array_uri, const tiledb_array_schema_t* array_schema) { - // Sanity checks - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; + ensure_array_schema_is_valid(array_schema); // Check array name tiledb::sm::URI uri(array_uri); @@ -2283,7 +1672,7 @@ int32_t tiledb_array_create( if (uri.is_tiledb()) { auto& rest_client = ctx->context().rest_client(); throw_if_not_ok(rest_client.post_array_schema_to_rest( - uri, *(array_schema->array_schema_.get()))); + uri, *(array_schema->array_schema()))); } else { // Create key tiledb::sm::EncryptionKey key; @@ -2291,21 +1680,20 @@ int32_t tiledb_array_create( key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the array tiledb::sm::Array::create( - ctx->resources(), uri, array_schema->array_schema_, key); + ctx->resources(), uri, array_schema->array_schema(), key); // Create any dimension labels in the array. for (tiledb::sm::ArraySchema::dimension_label_size_type ilabel{0}; - ilabel < array_schema->array_schema_->dim_label_num(); + ilabel < array_schema->dim_label_num(); ++ilabel) { // Get dimension label information and define URI and name. - const auto& dim_label_ref = - array_schema->array_schema_->dimension_label(ilabel); + const auto& dim_label_ref = array_schema->dimension_label(ilabel); if (dim_label_ref.is_external()) continue; if (!dim_label_ref.has_schema()) { - throw StatusException( - Status_Error("Failed to create array. Dimension labels that are " - "not external must have a schema.")); + throw CAPIStatusException( + "Failed to create array. Dimension labels that are " + "not external must have a schema."); } // Create the dimension label array with the same key. @@ -3044,8 +2432,7 @@ int32_t tiledb_serialize_array_schema( int32_t client_side, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx, array_schema) == TILEDB_ERR) - return TILEDB_ERR; + ensure_array_schema_is_valid(array_schema); // Create buffer auto buf = tiledb_buffer_handle_t::make_handle(); @@ -3053,7 +2440,7 @@ int32_t tiledb_serialize_array_schema( if (SAVE_ERROR_CATCH( ctx, tiledb::sm::serialization::array_schema_serialize( - *(array_schema->array_schema_.get()), + *(array_schema->array_schema()), (tiledb::sm::SerializationType)serialize_type, &(buf->buffer()), client_side))) { @@ -3073,29 +2460,17 @@ int32_t tiledb_deserialize_array_schema( int32_t, tiledb_array_schema_t** array_schema) { api::ensure_buffer_is_valid(buffer); + ensure_output_pointer_is_valid(array_schema); // Create array schema struct - *array_schema = new (std::nothrow) tiledb_array_schema_t; - if (*array_schema == nullptr) { - auto st = Status_Error("Failed to allocate TileDB array schema object"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_OOM; - } + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); + auto shared_schema = tiledb::sm::serialization::array_schema_deserialize( + (tiledb::sm::SerializationType)serialize_type, + buffer->buffer(), + memory_tracker); - try { - auto memory_tracker = ctx->resources().create_memory_tracker(); - memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); - (*array_schema)->array_schema_ = - tiledb::sm::serialization::array_schema_deserialize( - (tiledb::sm::SerializationType)serialize_type, - buffer->buffer(), - memory_tracker); - } catch (...) { - delete *array_schema; - *array_schema = nullptr; - throw; - } + *array_schema = tiledb_array_schema_t::make_handle(shared_schema); return TILEDB_OK; } @@ -3894,12 +3269,13 @@ capi_return_t tiledb_handle_load_array_schema_request( tiledb_serialization_type_t serialization_type, const tiledb_buffer_t* request, tiledb_buffer_t* response) { + ensure_context_is_valid(ctx); if (sanity_check(ctx, array) == TILEDB_ERR) { - throw std::invalid_argument("Array paramter must be valid."); + throw CAPIStatusException("Array paramter must be valid."); } - - api::ensure_buffer_is_valid(request); - api::ensure_buffer_is_valid(response); + ensure_array_is_valid(array); + ensure_buffer_is_valid(request); + ensure_buffer_is_valid(response); auto load_schema_req = tiledb::sm::serialization::deserialize_load_array_schema_request( @@ -3911,7 +3287,7 @@ capi_return_t tiledb_handle_load_array_schema_request( } tiledb::sm::serialization::serialize_load_array_schema_response( - array->array_->array_schema_latest(), + *array->array_, static_cast(serialization_type), response->buffer()); @@ -3935,7 +3311,9 @@ capi_return_t tiledb_handle_load_enumerations_request( tiledb::sm::serialization::deserialize_load_enumerations_request( static_cast(serialization_type), request->buffer()); - auto enumerations = array->array_->get_enumerations(enumeration_names); + auto enumerations = array->array_->get_enumerations( + enumeration_names, + array->array_->opened_array()->array_schema_latest_ptr()); tiledb::sm::serialization::serialize_load_enumerations_response( enumerations, @@ -4506,21 +3884,14 @@ int32_t tiledb_fragment_info_get_array_schema( tiledb_fragment_info_t* fragment_info, uint32_t fid, tiledb_array_schema_t** array_schema) { - if (sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) { return TILEDB_ERR; - - // Create array schema - *array_schema = new (std::nothrow) tiledb_array_schema_t; - if (*array_schema == nullptr) { - auto st = Status_Error("Failed to allocate TileDB array schema object"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_OOM; } + ensure_output_pointer_is_valid(array_schema); auto&& array_schema_get = fragment_info->fragment_info_->get_array_schema(fid); - (*array_schema)->array_schema_ = array_schema_get; + *array_schema = tiledb_array_schema_t::make_handle(array_schema_get); return TILEDB_OK; } @@ -4732,25 +4103,6 @@ constexpr auto api_entry = tiledb::api::api_entry_with_context; /* ****************************** */ /* ENUMS TO/FROM STR */ /* ****************************** */ -CAPI_INTERFACE( - array_type_to_str, tiledb_array_type_t array_type, const char** str) { - return api_entry_plain( - array_type, str); -} - -CAPI_INTERFACE( - array_type_from_str, const char* str, tiledb_array_type_t* array_type) { - return api_entry_plain( - str, array_type); -} - -CAPI_INTERFACE(layout_to_str, tiledb_layout_t layout, const char** str) { - return api_entry_plain(layout, str); -} - -CAPI_INTERFACE(layout_from_str, const char* str, tiledb_layout_t* layout) { - return api_entry_plain(str, layout); -} CAPI_INTERFACE( encryption_type_to_str, @@ -4857,304 +4209,6 @@ CAPI_INTERFACE(as_built_dump, tiledb_string_t** out) { return api_entry_plain(out); } -/* ****************************** */ -/* ARRAY SCHEMA */ -/* ****************************** */ - -CAPI_INTERFACE( - array_schema_alloc, - tiledb_ctx_t* ctx, - tiledb_array_type_t array_type, - tiledb_array_schema_t** array_schema) { - return api_entry( - ctx, array_type, array_schema); -} - -CAPI_INTERFACE_VOID(array_schema_free, tiledb_array_schema_t** array_schema) { - return api_entry_void(array_schema); -} - -CAPI_INTERFACE( - array_schema_add_attribute, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_attribute_t* attr) { - return api_entry( - ctx, array_schema, attr); -} - -CAPI_INTERFACE( - array_schema_set_allows_dups, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - int allows_dups) { - return api_entry( - ctx, array_schema, allows_dups); -} - -CAPI_INTERFACE( - array_schema_get_allows_dups, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - int* allows_dups) { - return api_entry( - ctx, array_schema, allows_dups); -} - -CAPI_INTERFACE( - array_schema_get_version, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - uint32_t* version) { - return api_entry( - ctx, array_schema, version); -} - -CAPI_INTERFACE( - array_schema_set_domain, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_domain_t* domain) { - return api_entry( - ctx, array_schema, domain); -} - -CAPI_INTERFACE( - array_schema_set_capacity, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - uint64_t capacity) { - return api_entry( - ctx, array_schema, capacity); -} - -CAPI_INTERFACE( - array_schema_set_cell_order, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_layout_t cell_order) { - return api_entry( - ctx, array_schema, cell_order); -} - -CAPI_INTERFACE( - array_schema_set_tile_order, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_layout_t tile_order) { - return api_entry( - ctx, array_schema, tile_order); -} - -CAPI_INTERFACE( - array_schema_timestamp_range, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - uint64_t* lo, - uint64_t* hi) { - return api_entry( - ctx, array_schema, lo, hi); -} - -CAPI_INTERFACE( - array_schema_add_enumeration, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_enumeration_t* enumeration) { - return api_entry( - ctx, array_schema, enumeration); -} - -CAPI_INTERFACE( - array_schema_set_coords_filter_list, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) { - return api_entry( - ctx, array_schema, filter_list); -} - -CAPI_INTERFACE( - array_schema_set_offsets_filter_list, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) { - return api_entry( - ctx, array_schema, filter_list); -} - -CAPI_INTERFACE( - array_schema_set_validity_filter_list, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) { - return api_entry( - ctx, array_schema, filter_list); -} - -CAPI_INTERFACE( - array_schema_check, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema) { - return api_entry(ctx, array_schema); -} - -CAPI_INTERFACE( - array_schema_load, - tiledb_ctx_t* ctx, - const char* array_uri, - tiledb_array_schema_t** array_schema) { - return api_entry( - ctx, array_uri, array_schema); -} - -CAPI_INTERFACE( - array_schema_get_array_type, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_array_type_t* array_type) { - return api_entry( - ctx, array_schema, array_type); -} - -CAPI_INTERFACE( - array_schema_get_capacity, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - uint64_t* capacity) { - return api_entry( - ctx, array_schema, capacity); -} - -CAPI_INTERFACE( - array_schema_get_cell_order, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_layout_t* cell_order) { - return api_entry( - ctx, array_schema, cell_order); -} - -CAPI_INTERFACE( - array_schema_get_coords_filter_list, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) { - return api_entry( - ctx, array_schema, filter_list); -} - -CAPI_INTERFACE( - array_schema_get_offsets_filter_list, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) { - return api_entry( - ctx, array_schema, filter_list); -} - -CAPI_INTERFACE( - array_schema_get_validity_filter_list, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) { - return api_entry( - ctx, array_schema, filter_list); -} - -CAPI_INTERFACE( - array_schema_get_domain, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_domain_t** domain) { - return api_entry( - ctx, array_schema, domain); -} - -CAPI_INTERFACE( - array_schema_get_tile_order, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_layout_t* tile_order) { - return api_entry( - ctx, array_schema, tile_order); -} - -CAPI_INTERFACE( - array_schema_get_attribute_num, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - uint32_t* attribute_num) { - return api_entry( - ctx, array_schema, attribute_num); -} - -CAPI_INTERFACE( - array_schema_dump, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - FILE* out) { - return api_entry( - ctx, array_schema, out); -} - -CAPI_INTERFACE( - array_schema_dump_str, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_string_t** out) { - return api_entry( - ctx, array_schema, out); -} - -CAPI_INTERFACE( - array_schema_get_attribute_from_index, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - uint32_t index, - tiledb_attribute_t** attr) { - return api_entry( - ctx, array_schema, index, attr); -} - -CAPI_INTERFACE( - array_schema_get_attribute_from_name, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - const char* name, - tiledb_attribute_t** attr) { - return api_entry( - ctx, array_schema, name, attr); -} - -CAPI_INTERFACE( - array_schema_has_attribute, - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - const char* name, - int32_t* has_attr) { - return api_entry( - ctx, array_schema, name, has_attr); -} - -CAPI_INTERFACE( - array_schema_set_current_domain, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_current_domain_t* current_domain) { - return api_entry( - ctx, array_schema, current_domain); -} - -CAPI_INTERFACE( - array_schema_get_current_domain, - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_current_domain_t** current_domain) { - return api_entry( - ctx, array_schema, current_domain); -} - /* ********************************* */ /* SCHEMA EVOLUTION */ /* ********************************* */ diff --git a/tiledb/sm/c_api/tiledb.h b/tiledb/sm/c_api/tiledb.h index 56baeaf1fc78..c0bd23eb93eb 100644 --- a/tiledb/sm/c_api/tiledb.h +++ b/tiledb/sm/c_api/tiledb.h @@ -55,6 +55,8 @@ /* * API sections */ +#include "tiledb/api/c_api/array/array_api_external.h" +#include "tiledb/api/c_api/array_schema/array_schema_api_external.h" #include "tiledb/api/c_api/attribute/attribute_api_external.h" #include "tiledb/api/c_api/buffer/buffer_api_external.h" #include "tiledb/api/c_api/buffer_list/buffer_list_api_external.h" @@ -121,22 +123,6 @@ typedef enum { #undef TILEDB_QUERY_CONDITION_COMBINATION_OP_ENUM } tiledb_query_condition_combination_op_t; -/** Array type. */ -typedef enum { -/** Helper macro for defining array type enums. */ -#define TILEDB_ARRAY_TYPE_ENUM(id) TILEDB_##id -#include "tiledb_enum.h" -#undef TILEDB_ARRAY_TYPE_ENUM -} tiledb_array_type_t; - -/** Tile or cell layout. */ -typedef enum { -/** Helper macro for defining layout type enums. */ -#define TILEDB_LAYOUT_ENUM(id) TILEDB_##id -#include "tiledb_enum.h" -#undef TILEDB_LAYOUT_ENUM -} tiledb_layout_t; - /** Encryption type. */ typedef enum { /** Helper macro for defining encryption enums. */ @@ -157,46 +143,6 @@ typedef enum { /* ENUMS TO/FROM STR */ /* ****************************** */ -/** - * Returns a string representation of the given array type. - * - * @param array_type Array type - * @param str Set to point to a constant string representation of the array type - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_type_to_str( - tiledb_array_type_t array_type, const char** str) TILEDB_NOEXCEPT; - -/** - * Parses a array type from the given string. - * - * @param str String representation to parse - * @param array_type Set to the parsed array type - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_type_from_str( - const char* str, tiledb_array_type_t* array_type) TILEDB_NOEXCEPT; - -/** - * Returns a string representation of the given layout. - * - * @param layout Layout - * @param str Set to point to a constant string representation of the layout - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t -tiledb_layout_to_str(tiledb_layout_t layout, const char** str) TILEDB_NOEXCEPT; - -/** - * Parses a layout from the given string. - * - * @param str String representation to parse - * @param layout Set to the parsed layout - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_layout_from_str( - const char* str, tiledb_layout_t* layout) TILEDB_NOEXCEPT; - /** * Returns a string representation of the given encryption type. * @@ -301,9 +247,6 @@ typedef struct tiledb_array_t tiledb_array_t; /** A subarray object. */ typedef struct tiledb_subarray_t tiledb_subarray_t; -/** A TileDB array schema. */ -typedef struct tiledb_array_schema_t tiledb_array_schema_t; - /** A TileDB query condition object. */ typedef struct tiledb_query_condition_t tiledb_query_condition_t; @@ -313,605 +256,6 @@ typedef struct tiledb_fragment_info_t tiledb_fragment_info_t; /** A consolidation plan object. */ typedef struct tiledb_consolidation_plan_t tiledb_consolidation_plan_t; -/* ********************************* */ -/* ARRAY SCHEMA */ -/* ********************************* */ - -/** - * Creates a TileDB array schema object. - * - * **Example:** - * - * @code{.c} - * tiledb_array_schema_t* array_schema; - * tiledb_array_schema_alloc(ctx, TILEDB_DENSE, &array_schema); - * @endcode - * - * @param ctx The TileDB context. - * @param array_type The array type. - * @param array_schema The TileDB array schema to be created. - * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_alloc( - tiledb_ctx_t* ctx, - tiledb_array_type_t array_type, - tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; - -/** - * Destroys an array schema, freeing associated memory. - * - * **Example:** - * - * @code{.c} - * tiledb_array_schema_free(&array_schema); - * @endcode - * - * @param array_schema The array schema to be destroyed. - */ -TILEDB_EXPORT void tiledb_array_schema_free( - tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; - -/** - * Adds an attribute to an array schema. - * - * **Example:** - * - * @code{.c} - * tiledb_attribute_t* attr; - * tiledb_attribute_alloc(ctx, "my_attr", TILEDB_INT32, &attr); - * tiledb_array_schema_add_attribute(ctx, array_schema, attr); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param attr The attribute to be added. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_add_attribute( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_attribute_t* attr) TILEDB_NOEXCEPT; - -/** - * Sets whether the array can allow coordinate duplicates or not. - * Applicable only to sparse arrays (it errors out if set to `1` for dense - * arrays). - * - * **Example:** - * - * @code{.c} - * int allows_dups = 1; - * tiledb_array_schema_set_allows_dups(ctx, array_schema, allows_dups); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param allows_dups Whether or not the array allows coordinate duplicates. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_set_allows_dups( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - int allows_dups) TILEDB_NOEXCEPT; - -/** - * Gets whether the array can allow coordinate duplicates or not. - * It should always be `0` for dense arrays. - * - * **Example:** - * - * @code{.c} - * int allows_dups; - * tiledb_array_schema_get_allows_dups(ctx, array_schema, &allows_dups); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param allows_dups Whether or not the array allows coordinate duplicates. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_allows_dups( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - int* allows_dups) TILEDB_NOEXCEPT; - -/** - * Returns the array schema version. - * - * **Example:** - * - * @code{.c} - * uint32_t version; - * tiledb_array_schema_get_version(ctx, array_schema, &version); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param version The version. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_version( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - uint32_t* version) TILEDB_NOEXCEPT; - -/** - * Sets a domain for the array schema. - * - * **Example:** - * - * @code{.c} - * tiledb_domain_t* domain; - * tiledb_domain_alloc(ctx, &domain); - * // -- Add dimensions to the domain here -- // - * tiledb_array_schema_set_domain(ctx, array_schema, domain); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param domain The domain to be set. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_set_domain( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_domain_t* domain) TILEDB_NOEXCEPT; - -/** - * Sets the tile capacity. Applies to sparse arrays only. - * - * **Example:** - * - * @code{.c} - * tiledb_array_schema_set_capacity(ctx, array_schema, 10000); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param capacity The capacity of a sparse data tile. Note that - * sparse data tiles exist in sparse fragments, which can be created - * in sparse arrays only. For more details, - * see [tutorials/tiling-sparse.html](tutorials/tiling-sparse.html). - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_set_capacity( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - uint64_t capacity) TILEDB_NOEXCEPT; - -/** - * Sets the cell order. - * - * **Example:** - * - * @code{.c} - * tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_ROW_MAJOR); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param cell_order The cell order to be set. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_set_cell_order( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_layout_t cell_order) TILEDB_NOEXCEPT; - -/** - * Sets the tile order. - * - * **Example:** - * - * @code{.c} - * tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_COL_MAJOR); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param tile_order The tile order to be set. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_set_tile_order( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_layout_t tile_order) TILEDB_NOEXCEPT; - -/** - * Sets the filter list to use for the coordinates. - * - * **Example:** - * - * @code{.c} - * tiledb_filter_list_t* filter_list; - * tiledb_filter_list_alloc(ctx, &filter_list); - * tiledb_filter_list_add_filter(ctx, filter_list, filter); - * tiledb_array_schema_set_coords_filter_list(ctx, array_schema, filter_list); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param filter_list The filter list to be set. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_set_coords_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) TILEDB_NOEXCEPT; - -/** - * Sets the filter list to use for the offsets of variable-sized attribute - * values. - * - * **Example:** - * - * @code{.c} - * tiledb_filter_list_t* filter_list; - * tiledb_filter_list_alloc(ctx, &filter_list); - * tiledb_filter_list_add_filter(ctx, filter_list, filter); - * tiledb_array_schema_set_offsets_filter_list(ctx, array_schema, filter_list); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param filter_list The filter list to be set. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_set_offsets_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) TILEDB_NOEXCEPT; - -/** - * Sets the filter list to use for the validity array of nullable attribute - * values. - * - * **Example:** - * - * @code{.c} - * tiledb_filter_list_t* filter_list; - * tiledb_filter_list_alloc(ctx, &filter_list); - * tiledb_filter_list_add_filter(ctx, filter_list, filter); - * tiledb_array_schema_set_validity_filter_list(ctx, array_schema, filter_list); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param filter_list The filter list to be set. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_set_validity_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) TILEDB_NOEXCEPT; - -/** - * Checks the correctness of the array schema. - * - * **Example:** - * - * @code{.c} - * tiledb_array_schema_check(ctx, array_schema); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @return `TILEDB_OK` if the array schema is correct and `TILEDB_ERR` upon any - * error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_check( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema) TILEDB_NOEXCEPT; - -/** - * Retrieves the schema of an array from the disk, creating an array schema - * struct. - * - * **Example:** - * - * @code{.c} - * tiledb_array_schema_t* array_schema; - * tiledb_array_schema_load(ctx, "s3://tiledb_bucket/my_array", &array_schema); - * // Make sure to free the array schema in the end - * @endcode - * - * @param ctx The TileDB context. - * @param array_uri The array whose schema will be retrieved. - * @param array_schema The array schema to be retrieved, or `NULL` upon error. - * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_load( - tiledb_ctx_t* ctx, - const char* array_uri, - tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; - -/** - * Retrieves the array type. - * - * **Example:** - * - * @code{.c} - * tiledb_array_schema_t* array_schema; - * tiledb_array_schema_load(ctx, "s3://tiledb_bucket/my_array", array_schema); - * tiledb_array_type_t* array_type; - * tiledb_array_schema_get_array_type(ctx, array_schema, &array_type); - * // Make sure to free the array schema in the end - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param array_type The array type to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_array_type( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_array_type_t* array_type) TILEDB_NOEXCEPT; - -/** - * Retrieves the capacity. - * - * **Example:** - * - * @code{.c} - * uint64_t capacity; - * tiledb_array_schema_get_capacity(ctx, array_schema, &capacity); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param capacity The capacity to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_capacity( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - uint64_t* capacity) TILEDB_NOEXCEPT; - -/** - * Retrieves the cell order. - * - * **Example:** - * - * @code{.c} - * tiledb_layout_t cell_order; - * tiledb_array_schema_get_cell_order(ctx, array_schema, &cell_order); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param cell_order The cell order to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_cell_order( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_layout_t* cell_order) TILEDB_NOEXCEPT; - -/** - * Retrieves the filter list used for the coordinates. - * - * **Example:** - * - * @code{.c} - * tiledb_filter_list_t* filter_list; - * tiledb_array_schema_get_coords_filter_list(ctx, array_schema, &filter_list); - * tiledb_filter_list_free(ctx, &filter_list); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param filter_list The filter list to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_coords_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) TILEDB_NOEXCEPT; - -/** - * Retrieves the filter list used for the offsets. - * - * **Example:** - * - * @code{.c} - * tiledb_filter_list_t* filter_list; - * tiledb_array_schema_get_offsets_filter_list(ctx, array_schema, &filter_list); - * tiledb_filter_list_free(ctx, &filter_list); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param filter_list The filter list to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_offsets_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) TILEDB_NOEXCEPT; - -/** - * Retrieves the filter list used for validity maps. - * - * **Example:** - * - * @code{.c} - * tiledb_filter_list_t* filter_list; - * tiledb_array_schema_get_validity_filter_list(ctx, array_schema, - * &filter_list); tiledb_filter_list_free(ctx, &filter_list); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param filter_list The filter list to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_validity_filter_list( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) TILEDB_NOEXCEPT; - -/** - * Retrieves the array domain. - * - * **Example:** - * - * @code{.c} - * tiledb_domain_t* domain; - * tiledb_array_schema_get_domain(ctx, array_schema, &domain); - * // Make sure to delete domain in the end - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param domain The array domain to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_domain( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_domain_t** domain) TILEDB_NOEXCEPT; - -/** - * Retrieves the tile order. - * - * **Example:** - * - * @code{.c} - * tiledb_layout_t tile_order; - * tiledb_array_schema_get_tile_order(ctx, array_schema, &tile_order); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param tile_order The tile order to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_tile_order( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_layout_t* tile_order) TILEDB_NOEXCEPT; - -/** - * Retrieves the number of array attributes. - * - * **Example:** - * - * @code{.c} - * uint32_t attr_num; - * tiledb_array_schema_get_attribute_num(ctx, array_schema, &attr_num); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param attribute_num The number of attributes to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_attribute_num( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - uint32_t* attribute_num) TILEDB_NOEXCEPT; - -/** - * Retrieves an attribute given its index. - * - * Attributes are ordered the same way they were defined - * when constructing the array schema. - * - * **Example:** - * - * The following retrieves the first attribute in the schema. - * - * @code{.c} - * tiledb_attribute_t* attr; - * tiledb_array_schema_get_attribute_from_index(ctx, array_schema, 0, &attr); - * // Make sure to delete the retrieved attribute in the end. - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param index The index of the attribute to retrieve. - * @param attr The attribute object to retrieve. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_attribute_from_index( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - uint32_t index, - tiledb_attribute_t** attr) TILEDB_NOEXCEPT; - -/** - * Retrieves an attribute given its name (key). - * - * **Example:** - * - * The following retrieves the first attribute in the schema. - * - * @code{.c} - * tiledb_attribute_t* attr; - * tiledb_array_schema_get_attribute_from_name( - * ctx, array_schema, "attr_0", &attr); - * // Make sure to delete the retrieved attribute in the end. - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param name The name (key) of the attribute to retrieve. - * @param attr THe attribute object to retrieve. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_attribute_from_name( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - const char* name, - tiledb_attribute_t** attr) TILEDB_NOEXCEPT; - -/** - * Checks whether the array schema has an attribute of the given name. - * - * **Example:** - * - * @code{.c} - * int32_t has_attr; - * tiledb_array_schema_has_attribute(ctx, array_schema, "attr_0", &has_attr); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param name The name of the attribute to check for. - * @param has_attr Set to `1` if the array schema has an attribute of the - * given name, else `0`. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_has_attribute( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - const char* name, - int32_t* has_attr) TILEDB_NOEXCEPT; - -/** - * Dumps the array schema in ASCII format in the selected string output. - * - * The output string handle must be freed by the user after use. - * - * **Example:** - * - * @code{.c} - * tiledb_string_t* tdb_string; - * tiledb_array_schema_dump_str(ctx, array_schema, &tdb_string); - * // Use the string - * tiledb_string_free(&tdb_string); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param out The output string. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_dump_str( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - tiledb_string_t** out) TILEDB_NOEXCEPT; - /* ********************************* */ /* QUERY */ /* ********************************* */ diff --git a/tiledb/sm/c_api/tiledb_deprecated.h b/tiledb/sm/c_api/tiledb_deprecated.h index c7c8379a9d60..d3bc83291b08 100644 --- a/tiledb/sm/c_api/tiledb_deprecated.h +++ b/tiledb/sm/c_api/tiledb_deprecated.h @@ -36,33 +36,14 @@ #include #include +#include "tiledb/api/c_api/array_schema/array_schema_api_deprecated.h" + #ifdef __cplusplus extern "C" { #endif // No deprecated APIs are declared at the moment. -/** - * Dumps the array schema in ASCII format in the selected file output. - * - * **Example:** - * - * The following prints the array schema dump in standard output. - * - * @code{.c} - * tiledb_array_schema_dump(ctx, array_schema, stdout); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param out The output handle. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_schema_dump( - tiledb_ctx_t* ctx, - const tiledb_array_schema_t* array_schema, - FILE* out) TILEDB_NOEXCEPT; - #ifdef __cplusplus } #endif diff --git a/tiledb/sm/c_api/tiledb_dimension_label.cc b/tiledb/sm/c_api/tiledb_dimension_label.cc index 01f472bf72a2..53712a508c10 100644 --- a/tiledb/sm/c_api/tiledb_dimension_label.cc +++ b/tiledb/sm/c_api/tiledb_dimension_label.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ * THE SOFTWARE. */ +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" #include "tiledb/api/c_api/attribute/attribute_api_internal.h" #include "tiledb/api/c_api/dimension/dimension_api_internal.h" #include "tiledb/api/c_api/dimension_label/dimension_label_api_internal.h" @@ -38,16 +39,13 @@ namespace tiledb::api { capi_return_t tiledb_array_schema_add_dimension_label( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, const uint32_t dim_index, const char* name, tiledb_data_order_t label_order, tiledb_datatype_t label_type) { - if (sanity_check(ctx, array_schema)) { - return TILEDB_ERR; - } - array_schema->array_schema_->add_dimension_label( + ensure_array_schema_is_valid(array_schema); + array_schema->add_dimension_label( dim_index, name, static_cast(label_order), @@ -56,17 +54,13 @@ capi_return_t tiledb_array_schema_add_dimension_label( } capi_return_t tiledb_array_schema_get_dimension_label_from_name( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, const char* label_name, tiledb_dimension_label_t** dim_label) { - if (sanity_check(ctx, array_schema)) { - return TILEDB_ERR; - } + ensure_array_schema_is_valid(array_schema); tiledb::api::ensure_output_pointer_is_valid(dim_label); *dim_label = tiledb_dimension_label_t::make_handle( - array_schema->array_schema_->array_uri(), - array_schema->array_schema_->dimension_label(label_name)); + array_schema->array_uri(), array_schema->dimension_label(label_name)); return TILEDB_OK; } @@ -74,42 +68,39 @@ capi_return_t tiledb_array_schema_has_dimension_label( const tiledb_array_schema_t* array_schema, const char* name, int32_t* has_dim_label) { - bool is_dim_label = array_schema->array_schema_->is_dim_label(name); + ensure_array_schema_is_valid(array_schema); + bool is_dim_label = array_schema->is_dim_label(name); *has_dim_label = is_dim_label ? 1 : 0; return TILEDB_OK; } capi_return_t tiledb_array_schema_set_dimension_label_filter_list( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, const char* label_name, tiledb_filter_list_t* filter_list) { - if (sanity_check(ctx, array_schema)) { - return TILEDB_ERR; - } - api::ensure_filter_list_is_valid(filter_list); - array_schema->array_schema_->set_dimension_label_filter_pipeline( + ensure_array_schema_is_valid(array_schema); + ensure_filter_list_is_valid(filter_list); + array_schema->set_dimension_label_filter_pipeline( label_name, filter_list->pipeline()); return TILEDB_OK; } capi_return_t tiledb_array_schema_set_dimension_label_tile_extent( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, const char* label_name, tiledb_datatype_t type, const void* tile_extent) { - if (sanity_check(ctx, array_schema)) { - return TILEDB_ERR; - } - array_schema->array_schema_->set_dimension_label_tile_extent( + ensure_array_schema_is_valid(array_schema); + array_schema->set_dimension_label_tile_extent( label_name, static_cast(type), tile_extent); return TILEDB_OK; } capi_return_t tiledb_array_schema_get_dimension_label_num( tiledb_array_schema_t* array_schema, uint64_t* dim_label_num) { - *dim_label_num = array_schema->array_schema_->dim_label_num(); + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(dim_label_num); + *dim_label_num = array_schema->dim_label_num(); return TILEDB_OK; } @@ -117,10 +108,11 @@ capi_return_t tiledb_array_schema_get_dimension_label_from_index( tiledb_array_schema_t* array_schema, uint64_t dim_label_index, tiledb_dimension_label_t** dim_label) { - tiledb::api::ensure_output_pointer_is_valid(dim_label); + ensure_array_schema_is_valid(array_schema); + ensure_output_pointer_is_valid(dim_label); *dim_label = tiledb_dimension_label_t::make_handle( - array_schema->array_schema_->array_uri(), - array_schema->array_schema_->dimension_label(dim_label_index)); + array_schema->array_uri(), + array_schema->dimension_label(dim_label_index)); return TILEDB_OK; } @@ -232,7 +224,8 @@ CAPI_INTERFACE( const char* name, tiledb_data_order_t label_order, tiledb_datatype_t label_type) { - return api_entry( + return api_entry_context< + tiledb::api::tiledb_array_schema_add_dimension_label>( ctx, array_schema, dim_index, name, label_order, label_type); } @@ -242,7 +235,7 @@ CAPI_INTERFACE( tiledb_array_schema_t* array_schema, const char* label_name, tiledb_dimension_label_t** dim_label) { - return api_entry_with_context< + return api_entry_context< tiledb::api::tiledb_array_schema_get_dimension_label_from_name>( ctx, array_schema, label_name, dim_label); } @@ -264,7 +257,7 @@ CAPI_INTERFACE( tiledb_array_schema_t* array_schema, const char* label_name, tiledb_filter_list_t* filter_list) { - return api_entry< + return api_entry_context< tiledb::api::tiledb_array_schema_set_dimension_label_filter_list>( ctx, array_schema, label_name, filter_list); } @@ -276,7 +269,7 @@ CAPI_INTERFACE( const char* label_name, tiledb_datatype_t type, const void* tile_extent) { - return api_entry< + return api_entry_context< tiledb::api::tiledb_array_schema_set_dimension_label_tile_extent>( ctx, array_schema, label_name, type, tile_extent); } diff --git a/tiledb/sm/c_api/tiledb_enum.h b/tiledb/sm/c_api/tiledb_enum.h index 9fb10866c03e..f70b36719cb2 100644 --- a/tiledb/sm/c_api/tiledb_enum.h +++ b/tiledb/sm/c_api/tiledb_enum.h @@ -1,7 +1,7 @@ /* * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,33 +31,13 @@ // clang-format is disabled on the first enum so that we can manually indent it // properly. // clang-format off -#ifdef TILEDB_ARRAY_TYPE_ENUM - /** Dense array */ - TILEDB_ARRAY_TYPE_ENUM(DENSE) = 0, - /** Sparse array */ - TILEDB_ARRAY_TYPE_ENUM(SPARSE) = 1, -#endif -// clang-format on - -#ifdef TILEDB_LAYOUT_ENUM - /** Row-major layout */ - TILEDB_LAYOUT_ENUM(ROW_MAJOR) = 0, - /** Column-major layout */ - TILEDB_LAYOUT_ENUM(COL_MAJOR) = 1, - /** Global-order layout */ - TILEDB_LAYOUT_ENUM(GLOBAL_ORDER) = 2, - /** Unordered layout */ - TILEDB_LAYOUT_ENUM(UNORDERED) = 3, - /** Hilbert layout */ - TILEDB_LAYOUT_ENUM(HILBERT) = 4, -#endif - #ifdef TILEDB_ENCRYPTION_TYPE_ENUM /** No encryption. */ TILEDB_ENCRYPTION_TYPE_ENUM(NO_ENCRYPTION) = 0, /** AES-256-GCM encryption. */ TILEDB_ENCRYPTION_TYPE_ENUM(AES_256_GCM) = 1, #endif +// clang-format on #ifdef TILEDB_QUERY_STATUS_ENUM /** Query failed */ diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index c95ee75d1bea..0817c6deb71b 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -5,8 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. - * @copyright Copyright (c) 2016 MIT and Intel Corporation + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -41,6 +40,7 @@ /* * API sections */ +#include "tiledb/api/c_api/array_schema/array_schema_api_experimental.h" #include "tiledb/api/c_api/attribute/attribute_api_external_experimental.h" #include "tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h" #include "tiledb/api/c_api/enumeration/enumeration_api_experimental.h" @@ -339,110 +339,6 @@ TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_expand_current_domain( tiledb_array_schema_evolution_t* array_schema_evolution, tiledb_current_domain_t* expanded_domain) TILEDB_NOEXCEPT; -/* ********************************* */ -/* ARRAY SCHEMA */ -/* ********************************* */ - -/** - * Gets timestamp range in an array schema evolution - * - * **Example:** - * - * @code{.c} - * uint64_t timestamp_lo = 0; - * uint64_t timestamp_hi = 0; - * tiledb_array_schema_evolution_timestamp_range(ctx, - * array_schema_evolution, ×tamp_lo, ×tamp_hi); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema object. - * @param lo The lower bound of timestamp range. - * @param hi The upper bound of timestamp range. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_timestamp_range( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - uint64_t* lo, - uint64_t* hi) TILEDB_NOEXCEPT; - -/** - * Adds an enumeration to an array schema. - * - * **Example:** - * - * @code{.c} - * tiledb_enumeration_t* enumeration; - * tiledb_enumeration_alloc( - * ctx, - * "enumeration_name", - * TILEDB_INT64, - * 1, - * FALSE, - * data, - * data_size, - * nullptr, - * 0, - * &enumeration); - * tiledb_array_schema_add_enumeration(ctx, array_schema, enumeration); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param enumeration The enumeration to add with the attribute - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; - -/** - * Sets the current domain on the array schema - * - * **Example:** - * - * @code{.c} - * tiledb_current_domain_t *current_domain; - * tiledb_current_domain_create(ctx, ¤t_domain); - * tiledb_array_schema_set_current_domain(ctx, array_schema, current_domain); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param current_domain The current domain to set on the schema - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_set_current_domain( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_current_domain_t* current_domain) TILEDB_NOEXCEPT; - -/** - * Gets the current domain set on the array schema or - * creates an empty current domain if none was set. - * It is the responsability of the caller to free the resources associated - * with the current domain when the handle isn't needed anymore. - * - * **Example:** - * - * @code{.c} - * tiledb_current_domain_t *current_domain; - * tiledb_array_schema_get_current_domain(ctx, array_schema, ¤t_domain); - * tiledb_current_domain_free(¤t_domain); - * @endcode - * - * @param ctx The TileDB context. - * @param array_schema The array schema. - * @param current_domain The current domain set on the schema - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_schema_get_current_domain( - tiledb_ctx_t* ctx, - tiledb_array_schema_t* array_schema, - tiledb_current_domain_t** current_domain) TILEDB_NOEXCEPT; - /* ********************************* */ /* ARRAY */ /* ********************************* */ diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index 13c513af4c8d..f0a67ce6d9aa 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -31,6 +31,7 @@ * This file defines the C API of TileDB for filestore. **/ +#include "tiledb/api/c_api/array_schema/array_schema_api_internal.h" #include "tiledb/api/c_api/attribute/attribute_api_internal.h" #include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/api/c_api/dimension/dimension_api_internal.h" @@ -111,7 +112,6 @@ int32_t tiledb_filestore_schema_create( } } - *array_schema = new tiledb_array_schema_t; try { // Share ownership of the internal ArraySchema ptr // All other calls for adding domains, attributes, etc @@ -119,9 +119,8 @@ int32_t tiledb_filestore_schema_create( // the cpp objects constructed here auto memory_tracker = context.resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_CREATE); - (*array_schema)->array_schema_ = make_shared( - HERE(), tiledb::sm::ArrayType::DENSE, memory_tracker); - auto& schema = (*array_schema)->array_schema_; + *array_schema = tiledb_array_schema_t::make_handle( + tiledb::sm::ArrayType::DENSE, memory_tracker); // Define the range of the dimension. uint64_t range_lo = 0; @@ -160,13 +159,14 @@ int32_t tiledb_filestore_schema_create( attr->set_filter_pipeline(filter); } - throw_if_not_ok(schema->set_domain(domain)); - throw_if_not_ok(schema->set_tile_order(tiledb::sm::Layout::ROW_MAJOR)); - throw_if_not_ok(schema->set_cell_order(tiledb::sm::Layout::ROW_MAJOR)); - throw_if_not_ok(schema->add_attribute(attr)); + throw_if_not_ok((*array_schema)->set_domain(domain)); + throw_if_not_ok( + (*array_schema)->set_tile_order(tiledb::sm::Layout::ROW_MAJOR)); + throw_if_not_ok( + (*array_schema)->set_cell_order(tiledb::sm::Layout::ROW_MAJOR)); + throw_if_not_ok((*array_schema)->add_attribute(attr)); } catch (const std::exception& e) { - (*array_schema)->array_schema_ = nullptr; - delete *array_schema; + tiledb_array_schema_t::break_handle(*array_schema); throw api::CAPIStatusException( std::string("Internal TileDB uncaught exception; ") + e.what()); } diff --git a/tiledb/sm/c_api/tiledb_struct_def.h b/tiledb/sm/c_api/tiledb_struct_def.h index b11364bda1c6..d9d755756092 100644 --- a/tiledb/sm/c_api/tiledb_struct_def.h +++ b/tiledb/sm/c_api/tiledb_struct_def.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -59,10 +59,6 @@ struct tiledb_subarray_t { bool is_allocated_ = false; }; -struct tiledb_array_schema_t { - shared_ptr array_schema_; -}; - struct tiledb_array_schema_evolution_t { tiledb::sm::ArraySchemaEvolution* array_schema_evolution_ = nullptr; }; diff --git a/tiledb/sm/c_api/tiledb_version.h b/tiledb/sm/c_api/tiledb_version.h index 54c76b536022..2ee548169b7b 100644 --- a/tiledb/sm/c_api/tiledb_version.h +++ b/tiledb/sm/c_api/tiledb_version.h @@ -27,5 +27,5 @@ */ #define TILEDB_VERSION_MAJOR 2 -#define TILEDB_VERSION_MINOR 26 +#define TILEDB_VERSION_MINOR 27 #define TILEDB_VERSION_PATCH 0 diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index 66f22432b038..107770cc2391 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -91,7 +91,7 @@ const std::string Config::REST_RETRY_DELAY_FACTOR = "1.25"; const std::string Config::REST_CURL_BUFFER_SIZE = "524288"; const std::string Config::REST_CAPNP_TRAVERSAL_LIMIT = "2147483648"; const std::string Config::REST_CURL_VERBOSE = "false"; -const std::string Config::REST_LOAD_ENUMERATIONS_ON_ARRAY_OPEN = "true"; +const std::string Config::REST_LOAD_ENUMERATIONS_ON_ARRAY_OPEN = "false"; const std::string Config::REST_LOAD_METADATA_ON_ARRAY_OPEN = "true"; const std::string Config::REST_LOAD_NON_EMPTY_DOMAIN_ON_ARRAY_OPEN = "true"; const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "true"; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index c72af74df97b..dd50be155d55 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -915,6 +915,10 @@ class Config { * If true, array non empty domain will be loaded and sent to server * together with the open array
* **Default**: true + * - `rest.load_enumerations_on_array_open`
+ * If true, enumerations will be loaded and sent to server together with + * the open array. + * **Default**: false * - `rest.use_refactored_array_open`
* If true, the new REST routes and APIs for opening an array will be used *
diff --git a/tiledb/sm/cpp_api/current_domain.h b/tiledb/sm/cpp_api/current_domain.h index f8456dce2761..249474d8be31 100644 --- a/tiledb/sm/cpp_api/current_domain.h +++ b/tiledb/sm/cpp_api/current_domain.h @@ -111,8 +111,8 @@ class CurrentDomain { tiledb_current_domain_type_t type() const { tiledb_current_domain_type_t type; auto& ctx = ctx_.get(); - ctx.handle_error( - tiledb_current_domain_get_type(current_domain_.get(), &type)); + ctx.handle_error(tiledb_current_domain_get_type( + ctx.ptr().get(), current_domain_.get(), &type)); return type; } @@ -126,7 +126,7 @@ class CurrentDomain { CurrentDomain& set_ndrectangle(const NDRectangle& ndrect) { auto& ctx = ctx_.get(); ctx.handle_error(tiledb_current_domain_set_ndrectangle( - current_domain_.get(), ndrect.ptr().get())); + ctx.ptr().get(), current_domain_.get(), ndrect.ptr().get())); return *this; } @@ -140,8 +140,8 @@ class CurrentDomain { NDRectangle ndrectangle() const { tiledb_ndrectangle_t* nd; auto& ctx = ctx_.get(); - ctx.handle_error( - tiledb_current_domain_get_ndrectangle(current_domain_.get(), &nd)); + ctx.handle_error(tiledb_current_domain_get_ndrectangle( + ctx.ptr().get(), current_domain_.get(), &nd)); return NDRectangle(ctx, nd); } @@ -153,8 +153,8 @@ class CurrentDomain { bool is_empty() const { uint32_t ret; auto& ctx = ctx_.get(); - ctx.handle_error( - tiledb_current_domain_get_is_empty(current_domain_.get(), &ret)); + ctx.handle_error(tiledb_current_domain_get_is_empty( + ctx.ptr().get(), current_domain_.get(), &ret)); return static_cast(ret); } diff --git a/tiledb/sm/cpp_api/ndrectangle.h b/tiledb/sm/cpp_api/ndrectangle.h index e0274fcdd0d1..9bb9d49df093 100644 --- a/tiledb/sm/cpp_api/ndrectangle.h +++ b/tiledb/sm/cpp_api/ndrectangle.h @@ -269,6 +269,52 @@ class NDRectangle { std::shared_ptr ptr() const { return ndrect_; } + /** + * Get the data type of the range at idx + * + * @param dim_idx The dimension index. + * @return The datatype of the range. + */ + tiledb_datatype_t range_dtype(unsigned dim_idx) { + auto& ctx = ctx_.get(); + + tiledb_datatype_t dtype; + ctx.handle_error(tiledb_ndrectangle_get_dtype( + ctx.ptr().get(), ndrect_.get(), dim_idx, &dtype)); + + return dtype; + } + + /** + * Get the data type of the range by name + * + * @param dim_name The dimension name. + * @return The datatype of the range. + */ + tiledb_datatype_t range_dtype(const std::string& dim_name) { + auto& ctx = ctx_.get(); + + tiledb_datatype_t dtype; + ctx.handle_error(tiledb_ndrectangle_get_dtype_from_name( + ctx.ptr().get(), ndrect_.get(), dim_name.c_str(), &dtype)); + + return dtype; + } + + /** + * Get the number of dimensions associated with the NDRectangle. + * + * @return The number of dimensions. + */ + uint32_t dim_num() { + auto& ctx = ctx_.get(); + + uint32_t ndim; + ctx.handle_error( + tiledb_ndrectangle_get_dim_num(ctx.ptr().get(), ndrect_.get(), &ndim)); + + return ndim; + } private: /* ********************************* */ diff --git a/tiledb/sm/crypto/CMakeLists.txt b/tiledb/sm/crypto/CMakeLists.txt index 1647dca95a05..5dd81745aa46 100644 --- a/tiledb/sm/crypto/CMakeLists.txt +++ b/tiledb/sm/crypto/CMakeLists.txt @@ -37,8 +37,9 @@ commence(object_library tiledb_crypto) this_target_link_libraries(bcrypt) else() find_package(OpenSSL REQUIRED) - #this_target_link_libraries(OpenSSL::Crypto) - target_link_libraries(tiledb_crypto PRIVATE OpenSSL::Crypto) + # We cannot use this_target_link_libraries, because it links with PUBLIC + # visibility, and we use OpenSSL only as an internal implementation detail. + target_link_libraries(tiledb_crypto PRIVATE OpenSSL::Crypto) endif() # OpenSSL-3 deprecates MD5 if(MSVC) @@ -47,3 +48,5 @@ commence(object_library tiledb_crypto) set_source_files_properties(crypto_openssl.cc PROPERTIES COMPILE_OPTIONS "-Wno-deprecated-declarations") endif() conclude(object_library) + +add_test_subdirectory() diff --git a/tiledb/sm/crypto/crypto.cc b/tiledb/sm/crypto/crypto.cc index 790d8b3e6540..1e7f4687b921 100644 --- a/tiledb/sm/crypto/crypto.cc +++ b/tiledb/sm/crypto/crypto.cc @@ -36,8 +36,10 @@ #ifdef _WIN32 #include "tiledb/sm/crypto/crypto_win32.h" +using PlatformCrypto = tiledb::sm::Win32CNG; #else #include "tiledb/sm/crypto/crypto_openssl.h" +using PlatformCrypto = tiledb::sm::OpenSSL; #endif using namespace tiledb::common; @@ -89,11 +91,7 @@ Status Crypto::decrypt_aes256gcm( return LOG_STATUS( Status_EncryptionError("AES-256-GCM error; invalid tag.")); -#ifdef _WIN32 - return Win32CNG::decrypt_aes256gcm(key, iv, tag, input, output); -#else - return OpenSSL::decrypt_aes256gcm(key, iv, tag, input, output); -#endif + return PlatformCrypto::decrypt_aes256gcm(key, iv, tag, input, output); } Status Crypto::md5(ConstBuffer* input, Buffer* output) { @@ -107,11 +105,7 @@ Status Crypto::md5( Status Crypto::md5( const void* input, uint64_t input_read_size, Buffer* output) { -#ifdef _WIN32 - return Win32CNG::md5(input, input_read_size, output); -#else - return OpenSSL::md5(input, input_read_size, output); -#endif + return PlatformCrypto::md5(input, input_read_size, output); } Status Crypto::sha256(ConstBuffer* input, Buffer* output) { @@ -125,11 +119,7 @@ Status Crypto::sha256( Status Crypto::sha256( const void* input, uint64_t input_read_size, Buffer* output) { -#ifdef _WIN32 - return Win32CNG::sha256(input, input_read_size, output); -#else - return OpenSSL::sha256(input, input_read_size, output); -#endif + return PlatformCrypto::sha256(input, input_read_size, output); } } // namespace sm diff --git a/tiledb/sm/crypto/crypto_openssl.cc b/tiledb/sm/crypto/crypto_openssl.cc index d5a64a4a5630..1d3234669a6a 100644 --- a/tiledb/sm/crypto/crypto_openssl.cc +++ b/tiledb/sm/crypto/crypto_openssl.cc @@ -48,20 +48,19 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -Status OpenSSL::get_random_bytes(unsigned num_bytes, Buffer* output) { - if (output->free_space() < num_bytes) - RETURN_NOT_OK(output->realloc(output->alloced_size() + num_bytes)); - - int rc = RAND_bytes((unsigned char*)output->cur_data(), num_bytes); - if (rc < 1) { - char err_msg[256]; - ERR_error_string_n(ERR_get_error(), err_msg, sizeof(err_msg)); - return Status_EncryptionError( - "Cannot generate random bytes with OpenSSL: " + std::string(err_msg)); +static Status get_random_bytes(span buffer) { + while (!buffer.empty()) { + int num_bytes = + int(std::min(buffer.size(), size_t(std::numeric_limits::max()))); + int rc = RAND_bytes(buffer.data(), num_bytes); + if (rc < 1) { + char err_msg[256]; + ERR_error_string_n(ERR_get_error(), err_msg, sizeof(err_msg)); + return Status_EncryptionError( + "Cannot generate random bytes with OpenSSL: " + std::string(err_msg)); + } + buffer = buffer.subspan(num_bytes); } - output->advance_size(num_bytes); - output->advance_offset(num_bytes); - return Status::Ok(); } @@ -83,11 +82,11 @@ Status OpenSSL::encrypt_aes256gcm( RETURN_NOT_OK(output->realloc(output->alloced_size() + required_space)); // Generate IV if the given IV buffer is null. - Buffer generated_iv; + std::array generated_iv; int iv_len; unsigned char* iv_buf; if (iv == nullptr || iv->data() == nullptr) { - RETURN_NOT_OK(get_random_bytes(Crypto::AES256GCM_IV_BYTES, &generated_iv)); + RETURN_NOT_OK(get_random_bytes(generated_iv)); iv_len = (int)generated_iv.size(); iv_buf = (unsigned char*)generated_iv.data(); } else { diff --git a/tiledb/sm/crypto/crypto_openssl.h b/tiledb/sm/crypto/crypto_openssl.h index 12a4d73d3a71..1151ab09f77e 100644 --- a/tiledb/sm/crypto/crypto_openssl.h +++ b/tiledb/sm/crypto/crypto_openssl.h @@ -107,16 +107,6 @@ class OpenSSL { */ static Status sha256( const void* input, uint64_t input_read_size, Buffer* output); - - private: - /** - * Generates a number of cryptographically random bytes. - * - * @param num_bytes Number of bytes to generate. - * @param output Buffer to store random bytes. - * @return Status - */ - static Status get_random_bytes(unsigned num_bytes, Buffer* output); }; } // namespace sm diff --git a/tiledb/sm/crypto/crypto_win32.cc b/tiledb/sm/crypto/crypto_win32.cc index acac8bec24ff..7a4287026775 100644 --- a/tiledb/sm/crypto/crypto_win32.cc +++ b/tiledb/sm/crypto/crypto_win32.cc @@ -33,6 +33,9 @@ #ifdef _WIN32 #include "tiledb/sm/crypto/crypto_win32.h" + +#include +#include #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" #include "tiledb/sm/buffer/buffer.h" @@ -47,28 +50,26 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -Status Win32CNG::get_random_bytes(unsigned num_bytes, Buffer* output) { - if (output->free_space() < num_bytes) - RETURN_NOT_OK(output->realloc(output->alloced_size() + num_bytes)); - +static Status get_random_bytes(span buffer) { BCRYPT_ALG_HANDLE alg_handle; if (!NT_SUCCESS(BCryptOpenAlgorithmProvider( &alg_handle, BCRYPT_RNG_ALGORITHM, nullptr, 0))) return Status_EncryptionError( "Win32CNG error; generating random bytes: error opening algorithm."); - if (!NT_SUCCESS(BCryptGenRandom( - alg_handle, (unsigned char*)output->cur_data(), num_bytes, 0))) { - BCryptCloseAlgorithmProvider(alg_handle, 0); - return Status_EncryptionError( - "Win32CNG error; generating random bytes: error generating bytes."); + while (!buffer.empty()) { + ULONG num_bytes = ULONG( + std::min(buffer.size(), size_t(std::numeric_limits::max()))); + if (!NT_SUCCESS(BCryptGenRandom(alg_handle, buffer.data(), num_bytes, 0))) { + BCryptCloseAlgorithmProvider(alg_handle, 0); + return Status_EncryptionError( + "Win32CNG error; generating random bytes: error generating bytes."); + } + buffer = buffer.subspan(num_bytes); } BCryptCloseAlgorithmProvider(alg_handle, 0); - output->advance_size(num_bytes); - output->advance_offset(num_bytes); - return Status::Ok(); } @@ -87,11 +88,11 @@ Status Win32CNG::encrypt_aes256gcm( // Generate IV if the given IV buffer is null. ULONG iv_len; unsigned char* iv_buf; - Buffer generated_iv; + std::array generated_iv; if (iv == nullptr || iv->data() == nullptr) { - RETURN_NOT_OK(get_random_bytes(Crypto::AES256GCM_IV_BYTES, &generated_iv)); + RETURN_NOT_OK(get_random_bytes(generated_iv)); iv_len = (ULONG)generated_iv.size(); - iv_buf = (unsigned char*)generated_iv.data(); + iv_buf = generated_iv.data(); } else { iv_len = (ULONG)iv->size(); iv_buf = (unsigned char*)iv->data(); @@ -294,16 +295,6 @@ Status Win32CNG::decrypt_aes256gcm( return Status::Ok(); } -Status Win32CNG::md5( - const void* input, uint64_t input_read_size, Buffer* output) { - return hash_bytes(input, input_read_size, output, BCRYPT_MD5_ALGORITHM); -} - -Status Win32CNG::sha256( - const void* input, uint64_t input_read_size, Buffer* output) { - return hash_bytes(input, input_read_size, output, BCRYPT_SHA256_ALGORITHM); -} - Status Win32CNG::hash_bytes( const void* input, uint64_t input_read_size, diff --git a/tiledb/sm/crypto/crypto_win32.h b/tiledb/sm/crypto/crypto_win32.h index 9ac945d19ccb..78446abc289a 100644 --- a/tiledb/sm/crypto/crypto_win32.h +++ b/tiledb/sm/crypto/crypto_win32.h @@ -41,6 +41,7 @@ #include #include + #include "tiledb/common/status.h" using namespace tiledb::common; @@ -101,7 +102,9 @@ class Win32CNG { * @return Status */ static Status md5( - const void* input, uint64_t input_read_size, Buffer* output); + const void* input, uint64_t input_read_size, Buffer* output) { + return hash_bytes(input, input_read_size, output, BCRYPT_MD5_ALGORITHM); + } /** * Compute sha256 checksum of data @@ -112,11 +115,14 @@ class Win32CNG { * @return Status */ static Status sha256( - const void* input, uint64_t input_read_size, Buffer* output); + const void* input, uint64_t input_read_size, Buffer* output) { + return hash_bytes(input, input_read_size, output, BCRYPT_SHA256_ALGORITHM); + } + private: /** * - * Compute a has using Win32CNG functions + * Compute a hash using Win32CNG functions * * @param input Plaintext to compute hash of * @param input_read_size size of input to read for hash @@ -129,16 +135,6 @@ class Win32CNG { uint64_t input_read_size, Buffer* output, LPCWSTR hash_algorithm); - - private: - /** - * Generates a number of cryptographically random bytes. - * - * @param num_bytes Number of bytes to generate. - * @param output Buffer to store random bytes. - * @return Status - */ - static Status get_random_bytes(unsigned num_bytes, Buffer* output); }; } // namespace sm diff --git a/tiledb/sm/crypto/test/CMakeLists.txt b/tiledb/sm/crypto/test/CMakeLists.txt new file mode 100644 index 000000000000..148bc59176a0 --- /dev/null +++ b/tiledb/sm/crypto/test/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# tiledb/sm/crypto/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test tiledb_crypto) + this_target_sources(unit_tiledb_crypto.cc) + this_target_object_libraries(tiledb_crypto) +conclude(unit_test) diff --git a/test/src/unit-crypto.cc b/tiledb/sm/crypto/test/unit_tiledb_crypto.cc similarity index 76% rename from test/src/unit-crypto.cc rename to tiledb/sm/crypto/test/unit_tiledb_crypto.cc index de90e64637b4..2ca086f4c0ce 100644 --- a/test/src/unit-crypto.cc +++ b/tiledb/sm/crypto/test/unit_tiledb_crypto.cc @@ -1,5 +1,5 @@ /** - * @file unit-crypto.cc + * @file unit_tiledb_crypto.cc * * @section LICENSE * @@ -29,12 +29,33 @@ * @section DESCRIPTION * * Tests the `Crypto` class + * + * @section Test plan + * + * The random number generator is tested by generating two 64-byte buffers with + * cryptographically random data and checking that their content is not the + * same. The probability of having the same content is vanishingly small. + * + * The algorithms are tested using official test vectors: + * + * AES-GCM test vectors are taken from gcmEncryptExtIV256.rsp, from "GCM Test + * Vectors (SP 800-38D)" + * (https://csrc.nist.gov/Projects/cryptographic-algorithm-validation-program/cavp-testing-block-cipher-modes). + * We also test that modifying the ciphertext causes failures in decrypting it. + * + * MD5 test vectors are taken from RFC 1321's section A.5 + * (https://www.ietf.org/rfc/rfc1321.txt) + * + * SHA-256 test vectors were taken from SHA256ShortMsg.rsp, from "SHA Test + * Vectors for Hashing Bit-Oriented Messages" + * (https://csrc.nist.gov/Projects/cryptographic-algorithm-validation-program/Secure-Hashing) */ #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/crypto/crypto.h" #include +#include #include using namespace tiledb::sm; @@ -214,28 +235,29 @@ TEST_CASE("Crypto: Test AES-256-GCM", "[crypto][aes]") { // These are test vectors where: // Keylen = 256, IVlen = 96, PTlen = 408, AADlen = 0, Taglen = 128. struct TestCase { - char key[64 + 1]; - char iv[24 + 1]; - char pt[102 + 1]; - char ct[102 + 1]; - char tag[32 + 1]; - - TestCase( - const char* key_arg, - const char* iv_arg, - const char* pt_arg, - const char* ct_arg, - const char* tag_arg) { - std::memcpy(key, key_arg, sizeof(key)); - std::memcpy(iv, iv_arg, sizeof(iv)); - std::memcpy(pt, pt_arg, sizeof(pt)); - std::memcpy(ct, ct_arg, sizeof(ct)); - std::memcpy(tag, tag_arg, sizeof(tag)); + span key; + span iv; + span plaintext; + span ciphertext; + span tag; + + constexpr TestCase( + decltype(key) key, + decltype(iv) iv, + decltype(plaintext) pt, + decltype(ciphertext) ct, + decltype(tag) tag) + : key(key) + , iv(iv) + , plaintext(pt) + , ciphertext(ct) + , tag(tag) { } - Buffer get_buffer( - unsigned buf_size, unsigned num_chars, const char* field) const { + static Buffer get_buffer(span field) { Buffer result; + auto num_chars = field.size() - 1; // Exclude null terminator. + auto buf_size = num_chars / 2; REQUIRE(result.realloc(buf_size).ok()); for (unsigned i = 0; i < num_chars; i += 2) { char byte_str[3] = {field[i], field[i + 1], '\0'}; @@ -246,27 +268,27 @@ TEST_CASE("Crypto: Test AES-256-GCM", "[crypto][aes]") { } Buffer get_key() const { - return get_buffer(256 / 8, 64, key); + return get_buffer(key); } Buffer get_iv() const { - return get_buffer(96 / 8, 24, iv); + return get_buffer(iv); } Buffer get_plaintext() const { - return get_buffer(408 / 8, 102, pt); + return get_buffer(plaintext); } - Buffer get_tag() const { - return get_buffer(128 / 8, 32, tag); + Buffer get_ciphertext() const { + return get_buffer(ciphertext); } - Buffer get_ciphertext() const { - return get_buffer(408 / 8, 102, ct); + Buffer get_tag() const { + return get_buffer(tag); } }; - std::vector tests = { + static auto tests = { TestCase( "1fded32d5999de4a76e0f8082108823aef60417e1896cf4218a2fa90f632ec8a", "1f3afa4711e9474f32e70462", @@ -493,50 +515,102 @@ TEST_CASE("Crypto: Test AES-256-GCM", "[crypto][aes]") { } } +static std::string from_hex(const std::string_view& str) { + std::string output; + for (size_t i = 0; i < str.length(); i += 2) { + char byte_str[3] = {str[i], str[i + 1], '\0'}; + output.push_back((uint8_t)std::strtoul(byte_str, nullptr, 16)); + } + return output; +} + +static std::string to_hex(span data) { + std::stringstream ss; + for (size_t i = 0; i < data.size(); i++) { + ss << std::hex << std::setw(2) << std::setfill('0') << (int)data[i]; + } + return ss.str(); +} + +/** + * Test that the given input (optionally in hex) has the expected hash value in + * hex. The function is generic over the hash and the input size. + */ +template +static void test_expected_hash_value( + const std::string_view& input, + const std::string_view& expected_value, + bool hex) { + REQUIRE(expected_value.length() == digest_bytes * 2); + + const std::string& processed_input = + hex ? from_hex(input) : std::string{input}; + + Buffer hash_buf(digest_bytes); + CHECK( + (hash(processed_input.data(), processed_input.length(), &hash_buf)).ok()); + + // Compare the strings for a better error message in case of failure. + CHECK( + to_hex( + {(uint8_t*)hash_buf.data(), + static_cast(hash_buf.alloced_size())}) == expected_value); +} + TEST_CASE("Crypto: Test MD5", "[crypto][md5]") { - SECTION("- Basic") { - std::string expected_checksum = "e99a18c428cb38d5f260853678922e03"; - std::string text_to_checksum = "abc123"; - ConstBuffer input_buffer( - text_to_checksum.data(), text_to_checksum.length()); - Buffer output_buffer; - CHECK(output_buffer.realloc(Crypto::MD5_DIGEST_BYTES).ok()); - CHECK(Crypto::md5(&input_buffer, &output_buffer).ok()); - - unsigned char* digest = - reinterpret_cast(output_buffer.data()); - char md5string[33]; - for (uint64_t i = 0; i < output_buffer.alloced_size(); ++i) { - sprintf(&md5string[i * 2], "%02x", (unsigned int)digest[i]); - } - CHECK( - memcmp( - expected_checksum.data(), md5string, expected_checksum.length()) == - 0); + auto test_md5 = + test_expected_hash_value; + static const std::vector> + test_cases{ + {"", "d41d8cd98f00b204e9800998ecf8427e"}, + {"a", "0cc175b9c0f1b6a831c399e269772661"}, + {"abc", "900150983cd24fb0d6963f7d28e17f72"}, + {"message digest", "f96b697d7cb7938d525a2f31aaf161d0"}, + {"abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b"}, + {"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "d174ab98d277d9f5a5611c2c9f419d9f"}, + {"1234567890123456789012345678901234567890123456789012345678901234567" + "890" + "1234567890", + "57edf4a22be3c955ac49da2e2107b67a"}}; + + for (auto& [input, expected_hash] : test_cases) { + test_md5(input, expected_hash, false); } } TEST_CASE("Crypto: Test SHA256", "[crypto][sha256]") { - SECTION("- Basic") { - std::string expected_checksum = - "6ca13d52ca70c883e0f0bb101e425a89e8624de51db2d2392593af6a84118090"; - std::string text_to_checksum = "abc123"; - ConstBuffer input_buffer( - text_to_checksum.data(), text_to_checksum.length()); - Buffer output_buffer; - CHECK(output_buffer.realloc(Crypto::SHA256_DIGEST_BYTES).ok()); - CHECK(Crypto::sha256(&input_buffer, &output_buffer).ok()); - - unsigned char* digest = - reinterpret_cast(output_buffer.data()); - char shastring[65]; - for (uint64_t i = 0; i < output_buffer.alloced_size(); ++i) { - sprintf(&shastring[i * 2], "%02x", (unsigned int)digest[i]); - } - - CHECK( - memcmp( - expected_checksum.data(), shastring, expected_checksum.length()) == - 0); + auto test_sha256 = + test_expected_hash_value; + std::vector> test_cases{ + // Len = 0 + {"", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + // Len = 64 + {"5738c929c4f4ccb6", + "963bb88f27f512777aab6c8b1a02c70ec0ad651d428f870036e1917120fb48bf"}, + // Len = 128 + {"0a27847cdc98bd6f62220b046edd762b", + "80c25ec1600587e7f28b18b1b18e3cdc89928e39cab3bc25e4d4a4c139bcedc4"}, + // Len = 192 + {"47991301156d1d977c0338efbcad41004133aefbca6bcf7e", + "feeb4b2b59fec8fdb1e55194a493d8c871757b5723675e93d3ac034b380b7fc9"}, + // Len = 256 + {"09fc1accc230a205e4a208e64a8f204291f581a12756392da4b8c0cf5ef02b95", + "4f44c1c7fbebb6f9601829f3897bfd650c56fa07844be76489076356ac1886a4"}, + // Len = 384 + {"4eef5107459bddf8f24fc7656fd4896da8711db50400c0164847f692b886ce8d7f4d67" + "395090b3534efd7b0d298da34b", + "7c5d14ed83dab875ac25ce7feed6ef837d58e79dc601fb3c1fca48d4464e8b83"}, + // Len = 448 + {"2d52447d1244d2ebc28650e7b05654bad35b3a68eedc7f8515306b496d75f3e73385dd" + "1b002625024b81a02f2fd6dffb6e6d561cb7d0bd7a", + "cfb88d6faf2de3a69d36195acec2e255e2af2b7d933997f348e09f6ce5758360"}, + // Len = 512 + {"5a86b737eaea8ee976a0a24da63e7ed7eefad18a101c1211e2b3650c5187c2a8a65054" + "7208251f6d4237e661c7bf4c77f335390394c37fa1a9f9be836ac28509", + "42e61e174fbb3897d6dd6cef3dd2802fe67b331953b06114a65c772859dfc1aa"}}; + + for (auto& [input, expected_hash] : test_cases) { + test_sha256(input, expected_hash, true); } } diff --git a/tiledb/sm/enums/array_type.h b/tiledb/sm/enums/array_type.h index 33e3b7c19571..4ce78e3bb1b9 100644 --- a/tiledb/sm/enums/array_type.h +++ b/tiledb/sm/enums/array_type.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -39,13 +39,12 @@ #include "tiledb/common/status.h" #include "tiledb/sm/misc/constants.h" -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Defines the array type. */ enum class ArrayType : uint8_t { #define TILEDB_ARRAY_TYPE_ENUM(id) id -#include "tiledb/sm/c_api/tiledb_enum.h" +#include "tiledb/api/c_api/array_schema/array_type_enum.h" #undef TILEDB_ARRAY_TYPE_ENUM }; @@ -83,7 +82,6 @@ inline void ensure_array_type_is_valid(uint8_t array_type_enum) { } } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_ARRAY_TYPE_H diff --git a/tiledb/sm/enums/layout.h b/tiledb/sm/enums/layout.h index 796519184a22..4f52af2b5d30 100644 --- a/tiledb/sm/enums/layout.h +++ b/tiledb/sm/enums/layout.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -41,13 +41,12 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Defines a layout for the cell or tile order. */ enum class Layout : uint8_t { #define TILEDB_LAYOUT_ENUM(id) id -#include "tiledb/sm/c_api/tiledb_enum.h" +#include "tiledb/api/c_api/array_schema/layout_enum.h" #undef TILEDB_LAYOUT_ENUM }; @@ -101,7 +100,6 @@ inline void ensure_cell_order_is_valid(uint8_t layout_enum) { "[Cell order] Invalid Layout enum " + std::to_string(layout_enum)); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_LAYOUT_H diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index c6af58a93ec0..3c8a5fe39871 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -1348,7 +1348,7 @@ Status S3::init_client() const { // check for client configuration on create, which can be slow if aws is not // configured on a users systems due to ec2 metadata check - client_config_ = make_shared(HERE()); + client_config_ = make_shared(HERE()); s3_tp_executor_ = make_shared(HERE(), vfs_thread_pool_); @@ -1388,6 +1388,10 @@ Status S3::init_client() const { s3_params_.connect_max_tries_, s3_params_.connect_scale_factor_); + client_config.useVirtualAddressing = s3_params_.use_virtual_addressing_; + client_config.payloadSigningPolicy = + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; + // If the user says not to sign a request, use the // AnonymousAWSCredentialsProvider This is equivalent to --no-sign-request on // the aws cli @@ -1443,7 +1447,12 @@ Status S3::init_client() const { session_name, external_id, load_frequency, - make_shared(HERE(), client_config)); + make_shared( + HERE(), + // client_config is an S3ClientConfiguration&, but gets + // casted to ClientConfiguration and copied to the STS + // client configuration. + Aws::STS::STSClientConfiguration(client_config))); break; } case 7: { @@ -1465,9 +1474,16 @@ Status S3::init_client() const { HERE(), Aws::Auth::GetConfigProfileName(), std::chrono::minutes(60), - [client_config](const auto& credentials) { + [config = this->client_config_](const auto& credentials) { return make_shared( - HERE(), credentials, client_config); + HERE(), + credentials, + // Create default endpoint provider. + make_shared(HERE()), + // *config is an S3ClientConfiguration&, but gets + // casted to ClientConfiguration and copied to the STS + // client configuration. + Aws::STS::STSClientConfiguration(*config)); }); break; } @@ -1492,20 +1508,11 @@ Status S3::init_client() const { { std::lock_guard static_lck(static_client_init_mtx); if (credentials_provider_ == nullptr) { - client_ = make_shared( - HERE(), - s3_params_, - *client_config_, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - s3_params_.use_virtual_addressing_); + client_ = + make_shared(HERE(), s3_params_, *client_config_); } else { client_ = make_shared( - HERE(), - s3_params_, - credentials_provider_, - *client_config_, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - s3_params_.use_virtual_addressing_); + HERE(), s3_params_, credentials_provider_, *client_config_); } } @@ -1786,12 +1793,6 @@ void S3::write_direct(const URI& uri, const void* buffer, uint64_t length) { put_object_request.SetBody(stream); put_object_request.SetContentLength(length); - // we only want to hash once, and must do it after setting the body - auto md5_hash = - Aws::Utils::HashingUtils::CalculateMD5(*put_object_request.GetBody()); - - put_object_request.SetContentMD5( - Aws::Utils::HashingUtils::Base64Encode(md5_hash)); put_object_request.SetContentType("application/octet-stream"); put_object_request.SetBucket(aws_uri.GetAuthority()); put_object_request.SetKey(aws_uri.GetPath()); @@ -1815,16 +1816,6 @@ void S3::write_direct(const URI& uri, const void* buffer, uint64_t length) { outcome_error_message(put_object_outcome)); } - // verify the MD5 hash of the result - // note the etag is hex-encoded not base64 - Aws::StringStream md5_hex; - md5_hex << "\"" << Aws::Utils::HashingUtils::HexEncode(md5_hash) << "\""; - if (md5_hex.str() != put_object_outcome.GetResult().GetETag()) { - throw S3Exception( - "Object uploaded successfully, but MD5 hash does " - "not match result from server!' "); - } - throw_if_not_ok(wait_for_object_to_propagate( put_object_request.GetBucket(), put_object_request.GetKey())); } @@ -1974,8 +1965,6 @@ S3::MakeUploadPartCtx S3::make_upload_part_req( upload_part_request.SetPartNumber(upload_part_num); upload_part_request.SetUploadId(upload_id); upload_part_request.SetBody(stream); - upload_part_request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode( - Aws::Utils::HashingUtils::CalculateMD5(*stream))); upload_part_request.SetContentLength(length); if (request_payer_ != Aws::S3::Model::RequestPayer::NOT_SET) upload_part_request.SetRequestPayer(request_payer_); diff --git a/tiledb/sm/filesystem/s3.h b/tiledb/sm/filesystem/s3.h index deb37c66249f..7d7a5cd4b5ca 100644 --- a/tiledb/sm/filesystem/s3.h +++ b/tiledb/sm/filesystem/s3.h @@ -69,6 +69,7 @@ #include #include #include +#include #include #include #include @@ -354,21 +355,20 @@ class TileDBS3Client : public Aws::S3::S3Client { public: TileDBS3Client( const S3Parameters& s3_params, - const Aws::Client::ClientConfiguration& client_config, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, - bool use_virtual_addressing) - : Aws::S3::S3Client(client_config, sign_payloads, use_virtual_addressing) + const Aws::S3::S3ClientConfiguration& client_config) + : Aws::S3::S3Client(client_config) , params_(s3_params) { } TileDBS3Client( const S3Parameters& s3_params, const std::shared_ptr& creds, - const Aws::Client::ClientConfiguration& client_config, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, - bool use_virtual_addressing) + const Aws::S3::S3ClientConfiguration& client_config) : Aws::S3::S3Client( - creds, client_config, sign_payloads, use_virtual_addressing) + creds, + // Create default endpoint provider. + make_shared(HERE()), + client_config) , params_(s3_params) { } @@ -1352,7 +1352,7 @@ class S3 : FilesystemBase { mutable std::mutex client_init_mtx_; /** Configuration object used to initialize the client. */ - mutable shared_ptr client_config_; + mutable shared_ptr client_config_; /** The executor used by 'client_'. */ mutable shared_ptr s3_tp_executor_; diff --git a/tiledb/sm/filter/filter_pipeline.h b/tiledb/sm/filter/filter_pipeline.h index ece9fba6a5fe..1db5bcec53b7 100644 --- a/tiledb/sm/filter/filter_pipeline.h +++ b/tiledb/sm/filter/filter_pipeline.h @@ -288,8 +288,9 @@ class FilterPipeline { FilterPipeline* pipeline, const EncryptionKey& encryption_key); /** - * Checks if an attribute/dimension needs to be filtered in chunks or as a - * whole + * Checks if the offsets tiles of an attribute/dimension should be skipped + * from being written. This happens in filters that encode the offsets + * alongside the data. * * @param type Datatype of the input attribute/dimension * @param version Array schema version diff --git a/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc index 9d2717548445..5dd2d9d93f63 100644 --- a/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc +++ b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc @@ -34,6 +34,7 @@ #include #include +#include #include "../bit_width_reduction_filter.h" #include "filter_test_support.h" diff --git a/tiledb/sm/filter/test/unit_xor_pipeline.cc b/tiledb/sm/filter/test/unit_xor_pipeline.cc index 3f4e18a17ea7..d384743ebca2 100644 --- a/tiledb/sm/filter/test/unit_xor_pipeline.cc +++ b/tiledb/sm/filter/test/unit_xor_pipeline.cc @@ -34,6 +34,7 @@ #include #include +#include #include "../bit_width_reduction_filter.h" #include "../bitshuffle_filter.h" diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 5cf2cf3ded44..236cb1e8de49 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -1938,22 +1938,19 @@ void FragmentMetadata::expand_non_empty_domain(const NDRange& mbr) { // ===== FORMAT ===== // bounding_coords_num (uint64_t) -// bounding_coords_#1 (void*) bounding_coords_#2 (void*) ... +// bounding_coords_#1 (dim_type[2][n_dims]) +// bounding_coords_#2 (dim_type[2][n_dims]) +// ... +// Note: This version supports only dimensions domains with the same type void FragmentMetadata::load_bounding_coords(Deserializer& deserializer) { // Get number of bounding coordinates - uint64_t bounding_coords_num = 0; - bounding_coords_num = deserializer.read(); + uint64_t bounding_coords_num = deserializer.read(); - // Get bounding coordinates - // Note: This version supports only dimensions domains with the same type auto coord_size{array_schema_->domain().dimension_ptr(0)->coord_size()}; auto dim_num = array_schema_->domain().dim_num(); uint64_t bounding_coords_size = 2 * dim_num * coord_size; - bounding_coords_.resize(bounding_coords_num); - for (uint64_t i = 0; i < bounding_coords_num; ++i) { - bounding_coords_[i].resize(bounding_coords_size); - deserializer.read(&bounding_coords_[i][0], bounding_coords_size); - } + // Skip bounding coordinates. They are not actually being used. + deserializer.skip(bounding_coords_num * bounding_coords_size); } void FragmentMetadata::load_file_sizes(Deserializer& deserializer) { @@ -2043,7 +2040,7 @@ void FragmentMetadata::load_has_timestamps(Deserializer& deserializer) { // ===== FORMAT ===== // has_delete_meta (char) void FragmentMetadata::load_has_delete_meta(Deserializer& deserializer) { - // Get includes timestamps + // Get includes delete metadata has_delete_meta_ = deserializer.read(); // Rebuild index map diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index b5ba41a8e325..931d6d0099fd 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -853,9 +853,6 @@ class FragmentMetadata { */ std::unordered_map idx_map_; - /** A vector storing the first and last coordinates of each tile. */ - std::vector> bounding_coords_; - /** True if the fragment is dense, and false if it is sparse. */ bool dense_; diff --git a/tiledb/sm/fragment/loaded_fragment_metadata.cc b/tiledb/sm/fragment/loaded_fragment_metadata.cc index 0bc0b7b1955d..b4e2776038f0 100644 --- a/tiledb/sm/fragment/loaded_fragment_metadata.cc +++ b/tiledb/sm/fragment/loaded_fragment_metadata.cc @@ -75,15 +75,16 @@ LoadedFragmentMetadata::LoadedFragmentMetadata( /* API */ /* ********************************* */ -LoadedFragmentMetadata* LoadedFragmentMetadata::create( +shared_ptr LoadedFragmentMetadata::create( FragmentMetadata& parent, shared_ptr memory_tracker, format_version_t version) { if (version <= 2) { - return tdb_new(V1V2PreloadedFragmentMetadata, parent, memory_tracker); + return make_shared( + HERE(), parent, memory_tracker); } - return tdb_new(OndemandFragmentMetadata, parent, memory_tracker); + return make_shared(HERE(), parent, memory_tracker); } uint64_t LoadedFragmentMetadata::persisted_tile_size( diff --git a/tiledb/sm/fragment/loaded_fragment_metadata.h b/tiledb/sm/fragment/loaded_fragment_metadata.h index b7d1845ebe8f..cd34362d73b5 100644 --- a/tiledb/sm/fragment/loaded_fragment_metadata.h +++ b/tiledb/sm/fragment/loaded_fragment_metadata.h @@ -95,7 +95,7 @@ class LoadedFragmentMetadata { * metadata corresponds to. * @param version The format version of the fragment */ - static LoadedFragmentMetadata* create( + static shared_ptr create( FragmentMetadata& parent, shared_ptr memory_tracker, format_version_t version); diff --git a/tiledb/sm/fragment/single_fragment_info.h b/tiledb/sm/fragment/single_fragment_info.h index 0b9126bb0c66..19a7dd21c9ca 100644 --- a/tiledb/sm/fragment/single_fragment_info.h +++ b/tiledb/sm/fragment/single_fragment_info.h @@ -125,6 +125,7 @@ class SingleFragmentInfo { std::stringstream ss; ss << " > URI: " << uri_.c_str() << "\n"; + ss << " > Schema name: " << array_schema_name_ << "\n"; ss << " > Type: " << (sparse_ ? "sparse" : "dense") << "\n"; ss << " > Non-empty domain: " << non_empty_domain_str(dim_types).c_str() << "\n"; diff --git a/tiledb/sm/misc/rectangle.cc b/tiledb/sm/misc/rectangle.cc index 7d21cad6ccbf..20782e6adb62 100644 --- a/tiledb/sm/misc/rectangle.cc +++ b/tiledb/sm/misc/rectangle.cc @@ -34,10 +34,6 @@ #include "tiledb/sm/misc/rectangle.h" -#ifdef __linux__ -#include "tiledb/sm/filesystem/posix.h" -#endif - using namespace tiledb::common; namespace tiledb::sm::rectangle { diff --git a/tiledb/sm/misc/rectangle.h b/tiledb/sm/misc/rectangle.h index 809dbbcf86f3..f188fdcc8857 100644 --- a/tiledb/sm/misc/rectangle.h +++ b/tiledb/sm/misc/rectangle.h @@ -33,6 +33,7 @@ #ifndef TILEDB_RECTANGLE_H #define TILEDB_RECTANGLE_H +#include #include #include "tiledb/common/status.h" diff --git a/tiledb/sm/query/query_condition.h b/tiledb/sm/query/query_condition.h index 6738063922b2..cd50ad61e92c 100644 --- a/tiledb/sm/query/query_condition.h +++ b/tiledb/sm/query/query_condition.h @@ -33,8 +33,7 @@ #ifndef TILEDB_QUERY_CONDITION_H #define TILEDB_QUERY_CONDITION_H -#include "external/include/span/span.hpp" - +#include #include #include "tiledb/common/status.h" diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index a6679c9c3eac..c916e65c717f 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -388,7 +388,7 @@ Status DenseReader::dense_read() { t_start, t_end, std::move(result_space_tiles), - *query_memory_tracker_); + query_memory_tracker_); // Add the number of cells to process to subarray_end_cell. for (uint64_t t = t_start; t < t_end; t++) { diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index 51abd0f43df9..2434839157ee 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -70,13 +70,14 @@ class DenseReader : public ReaderBase, public IQueryStrategy { uint64_t t_start, uint64_t t_end, std::map>&& result_space_tiles, - MemoryTracker& memory_tracker) + shared_ptr memory_tracker) : t_start_(t_start) , t_end_(t_end) + , memory_tracker_(memory_tracker) , tile_subarrays_( t_end - t_start, subarray.dim_num(), - memory_tracker.get_resource(MemoryType::DENSE_TILE_SUBARRAY)) + memory_tracker_->get_resource(MemoryType::DENSE_TILE_SUBARRAY)) , result_space_tiles_(std::move(result_space_tiles)) { auto& tile_coords = subarray.tile_coords(); throw_if_not_ok( @@ -127,6 +128,9 @@ class DenseReader : public ReaderBase, public IQueryStrategy { /** End tile to process. */ uint64_t t_end_; + /** Memory tracker. */ + shared_ptr memory_tracker_; + /** Tile subarrays. */ tdb::pmr::vector> tile_subarrays_; diff --git a/tiledb/sm/rest/curl.cc b/tiledb/sm/rest/curl.cc index f30ce80051fa..6b3f138b5b3d 100644 --- a/tiledb/sm/rest/curl.cc +++ b/tiledb/sm/rest/curl.cc @@ -238,10 +238,12 @@ size_t write_header_callback( const std::string header_value_domain = header_value_scheme_excl.substr(0, header_domain_end_pos); - const std::string redirection_value = - header_scheme + "://" + header_value_domain; - std::unique_lock rd_lck(*(pmHeader->redirect_uri_map_lock)); - (*pmHeader->redirect_uri_map)[*pmHeader->uri] = redirection_value; + if (pmHeader->should_cache_redirect) { + const std::string redirection_value = + header_scheme + "://" + header_value_domain; + std::unique_lock rd_lck(*(pmHeader->redirect_uri_map_lock)); + (*pmHeader->redirect_uri_map)[*pmHeader->uri] = redirection_value; + } } } @@ -262,7 +264,8 @@ Status Curl::init( const Config* config, const std::unordered_map& extra_headers, std::unordered_map* const res_headers, - std::mutex* const res_mtx) { + std::mutex* const res_mtx, + bool should_cache_redirect) { if (config == nullptr) return LOG_STATUS( Status_RestError("Error initializing libcurl; config is null.")); @@ -272,6 +275,7 @@ Status Curl::init( extra_headers_ = extra_headers; headerData.redirect_uri_map = res_headers; headerData.redirect_uri_map_lock = res_mtx; + headerData.should_cache_redirect = should_cache_redirect; // See https://curl.haxx.se/libcurl/c/threadsafe.html CURLcode rc = curl_easy_setopt(curl_.get(), CURLOPT_NOSIGNAL, 1); diff --git a/tiledb/sm/rest/curl.h b/tiledb/sm/rest/curl.h index 9defebb26428..47f8fc4b6dbe 100644 --- a/tiledb/sm/rest/curl.h +++ b/tiledb/sm/rest/curl.h @@ -85,6 +85,9 @@ struct HeaderCbData { /** A pointer to the lock attached to the shared resource of the cache map */ std::mutex* redirect_uri_map_lock; + + /** True if the uri should be stored in URI cache map, false if not */ + bool should_cache_redirect; }; /** @@ -128,7 +131,8 @@ class Curl { const Config* config, const std::unordered_map& extra_headers, std::unordered_map* res_headers, - std::mutex* res_mtx); + std::mutex* res_mtx, + bool should_cache_redirect = true); /** * Escapes the given URL. diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index c7237954309a..80d33d8a8411 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -317,7 +317,10 @@ class RestClient { } /// Operation disabled in base class. - inline virtual shared_ptr post_array_schema_from_rest( + inline virtual std::tuple< + shared_ptr, + std::unordered_map>> + post_array_schema_from_rest( const Config&, const URI&, uint64_t, uint64_t, bool) { throw RestClientDisabledException(); } diff --git a/tiledb/sm/rest/rest_client_remote.cc b/tiledb/sm/rest/rest_client_remote.cc index c108778129e0..afe69ea87a3d 100644 --- a/tiledb/sm/rest/rest_client_remote.cc +++ b/tiledb/sm/rest/rest_client_remote.cc @@ -226,13 +226,16 @@ RestClientRemote::get_array_schema_from_rest(const URI& uri) { serialization_type_, returned_data, memory_tracker_)}; } -shared_ptr RestClientRemote::post_array_schema_from_rest( +std::tuple< + shared_ptr, + std::unordered_map>> +RestClientRemote::post_array_schema_from_rest( const Config& config, const URI& uri, uint64_t timestamp_start, uint64_t timestamp_end, bool include_enumerations) { - serialization::LoadArraySchemaRequest req(include_enumerations); + serialization::LoadArraySchemaRequest req(config); Buffer buf; serialization::serialize_load_array_schema_request( @@ -271,7 +274,7 @@ shared_ptr RestClientRemote::post_array_schema_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); return serialization::deserialize_load_array_schema_response( - serialization_type_, returned_data, memory_tracker_); + uri, serialization_type_, returned_data, memory_tracker_); } Status RestClientRemote::post_array_schema_to_rest( @@ -296,8 +299,12 @@ Status RestClientRemote::post_array_schema_to_rest( std::string array_ns, array_uri; RETURN_NOT_OK(uri.get_rest_components(&array_ns, &array_uri)); const std::string cache_key = array_ns + ":" + array_uri; - RETURN_NOT_OK( - curlc.init(config_, extra_headers_, &redirect_meta_, &redirect_mtx_)); + // We don't want to cache the URI used for array creation as it will + // always be hardcoded to the default server. After creation the REST + // server knows the right region to direct the request to, so client + // side caching should start from then on. + RETURN_NOT_OK(curlc.init( + config_, extra_headers_, &redirect_meta_, &redirect_mtx_, false)); auto deduced_url = redirect_uri(cache_key) + "/v1/arrays/" + array_ns + "/" + curlc.url_escape(array_uri); Buffer returned_data; diff --git a/tiledb/sm/rest/rest_client_remote.h b/tiledb/sm/rest/rest_client_remote.h index 4ff2dc91a025..d40cdda2121d 100644 --- a/tiledb/sm/rest/rest_client_remote.h +++ b/tiledb/sm/rest/rest_client_remote.h @@ -151,10 +151,17 @@ class RestClientRemote : public RestClient { * get_array_schema_from_rest after TileDB-Cloud-REST merges support for the * POST endpoint. * + * @param config The TileDB config. * @param uri The Array URI to load the schema from. - * @return shared_ptr The loaded array schema. - */ - shared_ptr post_array_schema_from_rest( + * @param timestamp_start The starting timestamp used to open the array. + * @param timestamp_end The ending timestamp used to open the array. + * @return Tuple containing the latest array schema, and all array schemas for + * the array opened with provided timestamps. + */ + std::tuple< + shared_ptr, + std::unordered_map>> + post_array_schema_from_rest( const Config& config, const URI& uri, uint64_t timestamp_start, diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index b9d64cc84302..74c6c0e131b2 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -1850,6 +1850,8 @@ void load_array_schema_request_to_capnp( const LoadArraySchemaRequest& req) { auto config_builder = builder.initConfig(); throw_if_not_ok(config_to_capnp(config, &config_builder)); + // This boolean is only serialized to support clients using TileDB < 2.26. + // Future options should only be serialized within the Config object above. builder.setIncludeEnumerations(req.include_enumerations()); } @@ -1906,7 +1908,15 @@ void serialize_load_array_schema_request( LoadArraySchemaRequest load_array_schema_request_from_capnp( capnp::LoadArraySchemaRequest::Reader& reader) { - return LoadArraySchemaRequest(reader.getIncludeEnumerations()); + tdb_unique_ptr decoded_config = nullptr; + if (reader.hasConfig()) { + throw_if_not_ok(config_from_capnp(reader.getConfig(), &decoded_config)); + } else { + decoded_config.reset(tdb_new(Config)); + } + // We intentionally do not use the includeEnumerations field, as it is stored + // in the Config and set using the LoadArraySchemaRequest constructor. + return LoadArraySchemaRequest(*decoded_config); } LoadArraySchemaRequest deserialize_load_array_schema_request( @@ -1949,20 +1959,31 @@ LoadArraySchemaRequest deserialize_load_array_schema_request( } void load_array_schema_response_to_capnp( - capnp::LoadArraySchemaResponse::Builder& builder, - const ArraySchema& schema) { + capnp::LoadArraySchemaResponse::Builder& builder, const Array& array) { auto schema_builder = builder.initSchema(); - throw_if_not_ok(array_schema_to_capnp(schema, &schema_builder, false)); + throw_if_not_ok(array_schema_to_capnp( + array.array_schema_latest(), &schema_builder, false)); + + const auto& array_schemas_all = array.array_schemas_all(); + auto array_schemas_all_builder = builder.initArraySchemasAll(); + auto entries_builder = + array_schemas_all_builder.initEntries(array_schemas_all.size()); + uint64_t i = 0; + for (const auto& schema : array_schemas_all) { + auto entry = entries_builder[i++]; + entry.setKey(schema.first); + auto schema_entry_builder = entry.initValue(); + throw_if_not_ok( + array_schema_to_capnp(*(schema.second), &schema_entry_builder, false)); + } } void serialize_load_array_schema_response( - const ArraySchema& schema, - SerializationType serialization_type, - Buffer& data) { + const Array& array, SerializationType serialization_type, Buffer& data) { try { ::capnp::MallocMessageBuilder message; auto builder = message.initRoot(); - load_array_schema_response_to_capnp(builder, schema); + load_array_schema_response_to_capnp(builder, array); data.reset_size(); data.reset_offset(); @@ -2005,14 +2026,39 @@ void serialize_load_array_schema_response( } } -shared_ptr load_array_schema_response_from_capnp( +std::tuple< + shared_ptr, + std::unordered_map>> +load_array_schema_response_from_capnp( + const URI& uri, capnp::LoadArraySchemaResponse::Reader& reader, shared_ptr memory_tracker) { auto schema_reader = reader.getSchema(); - return array_schema_from_capnp(schema_reader, URI(), memory_tracker); + auto schema = array_schema_from_capnp(schema_reader, URI(), memory_tracker); + schema->set_array_uri(uri); + + std::unordered_map> all_schemas; + if (reader.hasArraySchemasAll()) { + auto all_schemas_reader = reader.getArraySchemasAll(); + + if (all_schemas_reader.hasEntries()) { + auto entries = all_schemas_reader.getEntries(); + for (auto array_schema_build : entries) { + auto schema_entry = array_schema_from_capnp( + array_schema_build.getValue(), schema->array_uri(), memory_tracker); + schema_entry->set_array_uri(schema->array_uri()); + all_schemas[array_schema_build.getKey()] = schema_entry; + } + } + } + return {schema, all_schemas}; } -shared_ptr deserialize_load_array_schema_response( +std::tuple< + shared_ptr, + std::unordered_map>> +deserialize_load_array_schema_response( + const URI& uri, SerializationType serialization_type, const Buffer& data, shared_ptr memory_tracker) { @@ -2026,7 +2072,8 @@ shared_ptr deserialize_load_array_schema_response( json.decode( kj::StringPtr(static_cast(data.data())), builder); auto reader = builder.asReader(); - return load_array_schema_response_from_capnp(reader, memory_tracker); + return load_array_schema_response_from_capnp( + uri, reader, memory_tracker); } case SerializationType::CAPNP: { const auto mBytes = reinterpret_cast(data.data()); @@ -2034,7 +2081,8 @@ shared_ptr deserialize_load_array_schema_response( reinterpret_cast(mBytes), data.size() / sizeof(::capnp::word))); auto reader = array_reader.getRoot(); - return load_array_schema_response_from_capnp(reader, memory_tracker); + return load_array_schema_response_from_capnp( + uri, reader, memory_tracker); } default: { throw ArraySchemaSerializationException( @@ -2115,12 +2163,15 @@ LoadArraySchemaRequest deserialize_load_array_schema_request( } void serialize_load_array_schema_response( - const ArraySchema&, SerializationType, Buffer&) { + const Array&, SerializationType, Buffer&) { throw ArraySchemaSerializationDisabledException(); } -shared_ptr deserialize_load_array_schema_response( - SerializationType, const Buffer&, shared_ptr) { +std::tuple< + shared_ptr, + std::unordered_map>> +deserialize_load_array_schema_response( + const URI&, SerializationType, const Buffer&, shared_ptr) { throw ArraySchemaSerializationDisabledException(); } diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index 14b51d7b6203..2fa3dff698b2 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -52,14 +52,16 @@ class Buffer; class ArraySchema; class Dimension; class MemoryTracker; +class URI; enum class SerializationType : uint8_t; namespace serialization { class LoadArraySchemaRequest { public: - LoadArraySchemaRequest(bool include_enumerations = false) - : include_enumerations_(include_enumerations) { + explicit LoadArraySchemaRequest(const Config& config) + : include_enumerations_(config.get( + "rest.load_enumerations_on_array_open", Config::must_find)) { } inline bool include_enumerations() const { @@ -208,11 +210,13 @@ LoadArraySchemaRequest deserialize_load_array_schema_request( SerializationType serialization_type, const Buffer& data); void serialize_load_array_schema_response( - const ArraySchema& schema, - SerializationType serialization_type, - Buffer& data); + const Array& array, SerializationType serialization_type, Buffer& data); -shared_ptr deserialize_load_array_schema_response( +std::tuple< + shared_ptr, + std::unordered_map>> +deserialize_load_array_schema_response( + const URI& uri, SerializationType serialization_type, const Buffer& data, shared_ptr memory_tracker); diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index a8a7fc1255e5..168cba4cdf9c 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -1267,11 +1267,16 @@ struct LoadArraySchemaRequest { includeEnumerations @1 :Bool; # When true, include all enumeration data in the returned ArraySchema + # This field is only serialized for backwards compatibility. Future options + # that modify array schema load behavior should be handled within the Config. } struct LoadArraySchemaResponse { schema @0 :ArraySchema; # The loaded ArraySchema + + arraySchemasAll @1 :Map(Text, ArraySchema); + # map of all Array Schemas } struct QueryPlanRequest { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index f04316412c56..bc5d31a2b56c 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -9797,17 +9797,17 @@ const ::capnp::_::RawSchema s_83f094010132ff21 = { 1, 2, i_83f094010132ff21, nullptr, nullptr, { &s_83f094010132ff21, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = { +static const ::capnp::_::AlignedData<69> b_ebe17f59ac9a1df1 = { { 0, 0, 0, 0, 5, 0, 6, 0, 241, 29, 154, 172, 89, 127, 225, 235, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 1, 0, 7, 0, 0, 0, 0, 0, + 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 82, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 63, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -9817,16 +9817,50 @@ static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = { 97, 82, 101, 115, 112, 111, 110, 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 4, 0, 0, 0, 3, 0, 4, 0, + 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 0, 0, 0, 58, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 0, 0, 0, 3, 0, 1, 0, - 20, 0, 0, 0, 2, 0, 1, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 128, 0, 0, 0, 2, 0, 1, 0, 115, 99, 104, 101, 109, 97, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, + 254, 150, 226, 152, 47, 227, 29, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 83, 99, 104, + 101, 109, 97, 115, 65, 108, 108, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 0, 0, 31, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 1, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 39, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, 254, 150, 226, 152, 47, 227, 29, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -9838,12 +9872,16 @@ static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = { #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_ebe17f59ac9a1df1[] = { &s_d71de32f98e296fe, + &s_f179c194ae71718c, +}; +static const uint16_t m_ebe17f59ac9a1df1[] = {1, 0}; +static const uint16_t i_ebe17f59ac9a1df1[] = {0, 1}; +KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_ebe17f59ac9a1df1[] = { + { 16777217, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::_capnpPrivate::brand() }, }; -static const uint16_t m_ebe17f59ac9a1df1[] = {0}; -static const uint16_t i_ebe17f59ac9a1df1[] = {0}; const ::capnp::_::RawSchema s_ebe17f59ac9a1df1 = { - 0xebe17f59ac9a1df1, b_ebe17f59ac9a1df1.words, 35, d_ebe17f59ac9a1df1, m_ebe17f59ac9a1df1, - 1, 1, i_ebe17f59ac9a1df1, nullptr, nullptr, { &s_ebe17f59ac9a1df1, nullptr, nullptr, 0, 0, nullptr }, false + 0xebe17f59ac9a1df1, b_ebe17f59ac9a1df1.words, 69, d_ebe17f59ac9a1df1, m_ebe17f59ac9a1df1, + 2, 2, i_ebe17f59ac9a1df1, nullptr, nullptr, { &s_ebe17f59ac9a1df1, nullptr, bd_ebe17f59ac9a1df1, 0, sizeof(bd_ebe17f59ac9a1df1) / sizeof(bd_ebe17f59ac9a1df1[0]), nullptr }, true }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<49> b_e06f571aa93eb314 = { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index a528dd8bb156..518bab1543fd 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -9,7 +9,7 @@ #ifndef CAPNP_VERSION #error \ "CAPNP_VERSION is not defined, is capnp/generated-header-support.h missing?" -#elif CAPNP_VERSION != 1000001 +#elif CAPNP_VERSION != 1000002 #error \ "Version mismatch between generated code and library headers. You must use the same version of the Cap'n Proto compiler and library." #endif @@ -1684,7 +1684,7 @@ struct LoadArraySchemaResponse { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(ebe17f59ac9a1df1, 0, 1) + CAPNP_DECLARE_STRUCT_HEADER(ebe17f59ac9a1df1, 0, 2) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -15033,6 +15033,12 @@ class LoadArraySchemaResponse::Reader { inline ::tiledb::sm::serialization::capnp::ArraySchema::Reader getSchema() const; + inline bool hasArraySchemasAll() const; + inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Reader + getArraySchemasAll() const; + private: ::capnp::_::StructReader _reader; template @@ -15082,6 +15088,28 @@ class LoadArraySchemaResponse::Builder { inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::ArraySchema> disownSchema(); + inline bool hasArraySchemasAll(); + inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Builder + getArraySchemasAll(); + inline void setArraySchemasAll( + ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Reader value); + inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Builder + initArraySchemasAll(); + inline void adoptArraySchemasAll( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>> + disownArraySchemasAll(); + private: ::capnp::_::StructBuilder _builder; template @@ -15104,6 +15132,10 @@ class LoadArraySchemaResponse::Pipeline { } inline ::tiledb::sm::serialization::capnp::ArraySchema::Pipeline getSchema(); + inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Pipeline + getArraySchemasAll(); private: ::capnp::AnyPointer::Pipeline _typeless; @@ -33186,6 +33218,80 @@ LoadArraySchemaResponse::Builder::disownSchema() { _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } +inline bool LoadArraySchemaResponse::Reader::hasArraySchemasAll() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool LoadArraySchemaResponse::Builder::hasArraySchemasAll() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp:: + Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::Reader + LoadArraySchemaResponse::Reader::getArraySchemasAll() const { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + get(_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp:: + Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::Builder + LoadArraySchemaResponse::Builder::getArraySchemasAll() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + get(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Pipeline +LoadArraySchemaResponse::Pipeline::getArraySchemasAll() { + return ::tiledb::sm::serialization::capnp:: + Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>:: + Pipeline(_typeless.getPointerField(1)); +} +#endif // !CAPNP_LITE +inline void LoadArraySchemaResponse::Builder::setArraySchemasAll( + ::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + set(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp:: + Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::Builder + LoadArraySchemaResponse::Builder::initArraySchemasAll() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + init(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void LoadArraySchemaResponse::Builder::adoptArraySchemasAll( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>> +LoadArraySchemaResponse::Builder::disownArraySchemasAll() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Map< + ::capnp::Text, + ::tiledb::sm::serialization::capnp::ArraySchema>>:: + disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + inline bool QueryPlanRequest::Reader::hasConfig() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); diff --git a/tiledb/storage_format/serialization/serializers.h b/tiledb/storage_format/serialization/serializers.h index 95506e0645b5..387abe764fa3 100644 --- a/tiledb/storage_format/serialization/serializers.h +++ b/tiledb/storage_format/serialization/serializers.h @@ -184,6 +184,21 @@ class Deserializer { size_ -= size; } + /** + * Advances the deserializer's pointer by a specified number of bytes, without + * reading them. + * + * @param size number of bytes to skip. + */ + void skip(storage_size_t size) { + if (size > size_) { + throw std::logic_error("Reading data past end of serialized data size."); + } + + ptr_ += size; + size_ -= size; + } + /** * Return remaining number of bytes to deserialize. * diff --git a/vcpkg.json b/vcpkg.json index 21f28a1eacd1..8b6d524d37cb 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,5 +1,5 @@ { - "builtin-baseline": "72010900b7cee36cea77aebb97695095c9358eaf", + "builtin-baseline": "7aeffc91033ad35cc4e2c152f213a866ec6c11ac", "dependencies": [ "bzip2", "libmagic", @@ -17,12 +17,7 @@ "description": "Support Azure Blob Storage", "dependencies": [ "azure-storage-blobs-cpp", - "azure-identity-cpp", - { - "name": "libxml2", - "default-features": false, - "platform": "!windows" - } + "azure-identity-cpp" ] }, "gcs": { @@ -68,8 +63,8 @@ "dependencies": [ "catch2", { - "name": "libfaketime", - "platform": "!windows" + "name": "libfaketime", + "platform": "!windows" } ] },